Merge pull request #202 from mendix/buildpack-2.2

Buildpack 2.2

Author: joshp123

lib/instadeploy.py

@@ -1,18 +1,20 @@
-from http.server import BaseHTTPRequestHandler, HTTPServer
 import cgi
 import json
-import shutil
-import time
 import logging
-import mxbuild
 import os
-from m2ee import logger
-import traceback
+import shutil
+import subprocess
 import threading
-import buildpackutil
+import time
+import traceback
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from urllib.parse import urljoin
+
 import requests
-import subprocess
 
+import buildpackutil
+import mxbuild
+from m2ee import logger
 
 ROOT_DIR = os.getcwd() + "/"
 MXBUILD_FOLDER = ROOT_DIR + "mxbuild/"
@@ -24,6 +26,8 @@
 INTERMEDIATE_MPK_DIR = ".local/tmp_project_2"
 MPK_FILE = os.path.join(PROJECT_DIR, "app.mpk")
 
+INSTADEPLOY_FEEDBACK_PATH = "api/1/environments/{app_id}/instadeployfeedback"
+
 for directory in (
     MXBUILD_FOLDER,
     PROJECT_DIR,
@@ -233,6 +237,13 @@ def build():
                 "Syncing project files failed: %s", traceback.format_exc()
             )
             raise
+        try:
+            send_metadata_to_cloudportal()
+        except Exception:
+            logger.warning(
+                "Failed to send instadeploy feedback to Cloud Portal",
+                exc_info=True,
+            )
     else:
         logger.warning("Not syncing project files. MxBuild result: %s", result)
 
@@ -261,3 +272,26 @@ def extract_mxbuild_response(mxbuild_response):
     if "message" in mxbuild_response:
         r["message"] = mxbuild_response["message"]
     return r
+
+
+def send_metadata_to_cloudportal():
+    required_variables = ("CLOUD_PORTAL_LOCATION", "ENVIRONMENT_SIGNATURE")
+
+    if not all([v in os.environ for v in required_variables]):
+        logger.info(
+            "Required environment variables for intadeploy feedback "
+            "are missing. Skipping instadeploy feedback."
+        )
+        return
+
+    app_id = json.loads(os.environ["VCAP_APPLICATION"])["name"]
+    target_path = INSTADEPLOY_FEEDBACK_PATH.format(app_id=app_id)
+    target_url = urljoin(os.environ["CLOUD_PORTAL_LOCATION"], target_path)
+
+    with open("/home/vcap/app/model/metadata.json", "rb") as metadata_json:
+        files = {"file": metadata_json}
+        headers = {
+            "Mendix-Environment-Signature": os.environ["ENVIRONMENT_SIGNATURE"]
+        }
+        # We don't care about the response.
+        requests.post(target_url, files=files, headers=headers)

lib/metrics.py

@@ -4,6 +4,7 @@
 import sys
 import threading
 import time
+
 from abc import ABCMeta, abstractmethod
 
 BUILDPACK_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@@ -157,7 +158,6 @@ def _inject_health(self, stats):
             logger.warn("Metrics: Failed to get health status, " + str(e))
             health["health"] = translation["critical"]
             health["diagnosis"] = "Health check failed unexpectedly: %s" % e
-
         return stats
 
     def _inject_m2ee_stats(self, stats):
@@ -285,6 +285,25 @@ def _get_number_of_files(self):
                 raise Exception("Unexpected result from database query")
             return int(rows[0][0])
 
+    def _get_size_of_files(self):
+        conn = self._get_db_conn()
+        with conn.cursor() as cursor:
+            try:
+                cursor.execute(
+                    "SELECT sum(size) from system$filedocument WHERE hascontents=true;"
+                )
+                rows = cursor.fetchall()
+                if len(rows) == 0:
+                    return 0
+                return int(rows[0][0])
+            except Exception as e:
+                # We ignore errors here, as the information is not available for
+                # older mendix versions
+                logger.debug(
+                    "METRICS: Error retrieving file sizes", exc_info=True
+                )
+                return 0
+
     def _get_db_conn(self):
         if self.db and self.db.closed != 0:
             self.db.close()

start.py

@@ -1,14 +1,17 @@
 #!/usr/bin/env python3
 import atexit
 import base64
+import codecs
 import datetime
+import itertools
 import json
 import logging
 import os
 import re
 import signal
 import subprocess
 import sys
+import threading
 import time
 import traceback
 import uuid
@@ -24,8 +27,32 @@
 from nginx import get_path_config, gen_htpasswd  # noqa: E402
 from buildpackutil import i_am_primary_instance  # noqa: E402
 
+HEARTBEAT_SOURCE_STRING = """Gur Mra bs Clguba, ol Gvz Crgref
+Ornhgvshy vf orggre guna htyl.
+Rkcyvpvg vf orggre guna vzcyvpvg.
+Fvzcyr vf orggre guna pbzcyrk.
+Pbzcyrk vf orggre guna pbzcyvpngrq.
+Syng vf orggre guna arfgrq.
+Fcnefr vf orggre guna qrafr.
+Ernqnovyvgl pbhagf.
+Fcrpvny pnfrf nera'g fcrpvny rabhtu gb oernx gur ehyrf.
+Nygubhtu cenpgvpnyvgl orngf chevgl.
+Reebef fubhyq arire cnff fvyragyl.
+Hayrff rkcyvpvgyl fvyraprq.
+Va gur snpr bs nzovthvgl, ershfr gur grzcgngvba gb thrff.
+Gurer fubhyq or bar-- naq cersrenoyl bayl bar --boivbhf jnl gb qb vg.
+Nygubhtu gung jnl znl abg or boivbhf ng svefg hayrff lbh'er Qhgpu.
+Abj vf orggre guna arire.
+Nygubhtu arire vf bsgra orggre guna *evtug* abj.
+Vs gur vzcyrzragngvba vf uneq gb rkcynva, vg'f n onq vqrn.
+Vs gur vzcyrzragngvba vf rnfl gb rkcynva, vg znl or n tbbq vqrn.
+Anzrfcnprf ner bar ubaxvat terng vqrn -- yrg'f qb zber bs gubfr!"""
+HEARTBEAT_STRING_LIST = codecs.encode(HEARTBEAT_SOURCE_STRING, "rot13").split(
+    "\n"
+)
+
 logger.setLevel(buildpackutil.get_buildpack_loglevel())
-logger.info("Started Mendix Cloud Foundry Buildpack v2.1.3")
+logger.info("Started Mendix Cloud Foundry Buildpack v2.2.1")
 logging.getLogger("m2ee").propagate = False
 
 
@@ -228,6 +255,14 @@ def get_constants(metadata):
     return constants
 
 
+def set_jvm_locale(m2ee_section, java_version):
+    javaopts = m2ee_section["javaopts"]
+
+    # enable locale for java8 or later
+    if not java_version.startswith("7"):
+        javaopts.append("-Djava.locale.providers=JRE,SPI,CLDR")
+
+
 def set_jvm_memory(m2ee_section, vcap, java_version):
     max_memory = os.environ.get("MEMORY_LIMIT")
 
@@ -720,6 +755,7 @@ def set_up_m2ee_client(vcap_data):
         m2ee.config.get_runtime_version()
     )
     set_jvm_memory(m2ee.config._conf["m2ee"], vcap_data, java_version)
+    set_jvm_locale(m2ee.config._conf["m2ee"], java_version)
     set_jetty_config(m2ee)
     activate_new_relic(m2ee, vcap_data["application_name"])
     activate_appdynamics(m2ee, vcap_data["application_name"])
@@ -818,7 +854,7 @@ def service_backups():
         logger.warning("Failed to contact backup service. SSLError: " + str(e))
         return
     except Exception as e:
-        logger.warning("Failed to contact backup service: " + e)
+        logger.warning("Failed to contact backup service: ", exc_info=True)
         return
 
     if result.status_code == 200:
@@ -1041,6 +1077,29 @@ def start_metrics(m2ee):
         thread.start()
 
 
+class LoggingHeartbeatEmitterThread(threading.Thread):
+    def __init__(self, interval):
+        super().__init__()
+        self.interval = interval
+
+    def run(self):
+        logger.debug(
+            "Starting metrics emitter with interval %d", self.interval
+        )
+        for line in itertools.cycle(HEARTBEAT_STRING_LIST):
+            logger.info("MENDIX-LOGGING-HEARTBEAT: %s", line)
+            time.sleep(self.interval)
+
+
+def start_logging_heartbeat():
+    logging_interval = os.getenv(
+        "METRICS_LOGGING_HEARTBEAT_INTERVAL", str(3600 * 6)
+    )
+    thread = LoggingHeartbeatEmitterThread(int(logging_interval))
+    thread.setDaemon(True)
+    thread.start()
+
+
 def complete_start_procedure_safe_to_use_for_restart(m2ee):
     buildpackutil.mkdir_p("model/lib/userlib")
     set_up_logging_file()
@@ -1089,6 +1148,7 @@ def sigusr_handler(_signo, _stack_frame):
         complete_start_procedure_safe_to_use_for_restart(m2ee)
         set_up_instadeploy_if_deploy_password_is_set(m2ee)
         start_metrics(m2ee)
+        start_logging_heartbeat()
         start_nginx()
         loop_until_process_dies(m2ee)
     except Exception: