Merge pull request #388 from davidkleiven/mysqlSupport

Add support for MySQL and MariaDB

Author: apdavison

sumatra/recordstore/django_store/__init__.py

@@ -2,7 +2,7 @@
 Handles storage of simulation/analysis records in a relational database, via the
 Django object-relational mapper (ORM), which means that any database
 supported by Django could in principle be used, although for now we assume
-SQLite or PostgreSQL.
+SQLite, PostgreSQL, MySQL or MariaDB.
 
 
 :copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt
@@ -72,6 +72,13 @@ def uri_to_db(self, uri):
             db['USER'] = parse_result.username
             db['PASSWORD'] = parse_result.password
             db['PORT'] = parse_result.port or ''
+        elif 'mysql' in parse_result.scheme or 'mariadb' in parse_result.scheme:
+            db['ENGINE'] = 'django.db.backends.mysql'
+            db['NAME'] = os.path.split(parse_result.path)[-1]
+            db['HOST'] = parse_result.hostname
+            db['USER'] = parse_result.username
+            db['PASSWORD'] = parse_result.password
+            db['PORT'] = parse_result.port or ''
         else:
             db['ENGINE'] = 'django.db.backends.sqlite3'
             db['NAME'] = os.path.abspath(parse_result.path)
@@ -342,7 +349,7 @@ def _dump(self, indent=2):
 
     @classmethod
     def accepts_uri(cls, uri):
-        return uri[:8] == "postgres" or os.path.exists(uri) or os.path.exists(uri + ".db")
+        return uri[:8] == "postgres" or uri.startswith(('mysql', 'mariadb')) or os.path.exists(uri) or os.path.exists(uri + ".db")
 
     def backup(self):
         """
@@ -351,7 +358,7 @@ def backup(self):
         if 'sqlite3' in db_config.engine:
             shutil.copy2(self._db_file, self._db_file + ".backup")
         else:
-            warn("Cannot backup a PostgreSQL store directly.")
+            warn("Cannot backup a PostgreSQL/MySQL/MariaDB store directly.")
 
     def remove(self):
         """
@@ -361,4 +368,4 @@ def remove(self):
         if 'sqlite3' in db_config.engine:
             os.remove(self._db_file)
         else:
-            warn("Cannot remove a PostgreSQL store directly.")
+            warn("Cannot remove a PostgreSQL/MySQL/MariaDB store directly.")

test/system/fixtures/Dockerfile.mysql

@@ -0,0 +1,20 @@
+FROM debian:jessie
+
+RUN apt-get update
+RUN apt-get -y -q install python-software-properties software-properties-common python-pip 
+ENV MYSQL_PWD Pwd123
+RUN echo "mysql-server mysql-server/root_password password $MYSQL_PWD" | debconf-set-selections
+RUN echo "mysql-server mysql-server/root_password_again password $MYSQL_PWD" | debconf-set-selections
+RUN apt-get -y install mysql-server
+
+ENV WORK=/mysqlwork
+WORKDIR ${WORK}
+
+RUN service mysql start &&\
+	mysql -u root --password=${MYSQL_PWD} -e "CREATE USER docker IDENTIFIED BY 'docker';" &&\
+	mysql -u root --password=${MYSQL_PWD} -e "CREATE DATABASE sumatra_test;" && \
+	mysql -u root --password=${MYSQL_PWD} -e "GRANT ALL PRIVILEGES ON sumatra_test.* TO docker IDENTIFIED BY 'docker';"
+
+EXPOSE 5432
+VOLUME ["/var/lib/mysql"]
+CMD ["mysqld"]

test/system/test_mysql.py

@@ -0,0 +1,120 @@
+"""
+Tests using a MySQL-based record store.
+
+
+Usage:
+    nosetests -v test_mysql.py
+or:
+    python test_mysql.py
+"""
+from __future__ import print_function
+from __future__ import unicode_literals
+from future import standard_library
+standard_library.install_aliases()
+from builtins import input
+
+import os
+from urllib.parse import urlparse
+try:
+    import docker
+    if "DOCKER_HOST" in os.environ:
+        have_docker = True
+    else:
+        have_docker = False
+except ImportError:
+    have_docker = False
+from unittest import SkipTest
+import utils
+from utils import setup, teardown as default_teardown, run_test, build_command
+
+ctr = None
+dkr = None
+image = "mysql_test"
+
+
+def create_script():
+    with open(os.path.join(utils.working_dir, "main.py"), "w") as fp:
+        fp.write("print('Hello world!')\n")
+
+
+def get_url():
+    info = dkr.containers()[0]
+    assert info["Image"] == image
+    host = urlparse(dkr.base_url).hostname
+    return "{0}:{1}".format(host, info["Ports"][0]["PublicPort"])
+
+
+def start_pg_container():
+    """
+    Launch a Docker container running MySQL, return the URL.
+
+    Requires the "mysql_test" image. If this does not yet exist, run
+
+        docker build -t mysql_test - < fixtures/Dockerfile.mysql
+    """
+    global ctr, dkr
+    env = docker.utils.kwargs_from_env(assert_hostname=False)
+    dkr = docker.Client(timeout=60, **env)
+
+    host_config = dkr.create_host_config(publish_all_ports=True)
+    # docker run -rm -P -name ms_test mysql_test
+    ctr = dkr.create_container(image, command=None, hostname=None, user=None,
+                               detach=False, stdin_open=False, tty=False,
+                               ports=None, environment=None, dns=None,
+                               volumes=None, volumes_from=None,
+                               network_disabled=False, name=None,
+                               entrypoint=None, cpu_shares=None,
+                               working_dir=None, host_config=host_config)
+    dkr.start(ctr)
+    utils.env["url"] = get_url()
+
+
+def teardown():
+    global ctr, dkr
+    default_teardown()
+    if dkr is not None:
+        dkr.stop(ctr)
+
+
+test_steps = [
+    create_script,
+    ("Create repository", "git init"),
+    ("Add main file", "git add main.py"),
+    ("Commit main file", "git commit -m 'initial'"),
+    start_pg_container,
+    ("Set up a Sumatra project",
+     build_command("smt init --store=mysql://docker:docker@{}/sumatra_test -m main.py -e python MyProject", "url")),
+    ("Show project configuration", "smt info"),  # TODO: add assert
+    ("Run a computation", "smt run")
+]
+
+
+# TODO: add test skips where docker not found
+
+
+def test_all():
+    """Test generator for Nose."""
+    if not have_docker:
+        raise SkipTest("Tests require docker")
+    for step in test_steps:
+        if callable(step):
+            step()
+        else:
+            run_test.description = step[0]
+            yield tuple([run_test] + list(step[1:]))
+
+
+if __name__ == '__main__':
+    # Run the tests without using Nose.
+    setup()
+    for step in test_steps:
+        if callable(step):
+            step()
+        else:
+            print(step[0])  # description
+            run_test(*step[1:])
+    response = input("Do you want to delete the temporary directory (default: yes)? ")
+    if response not in ["n", "N", "no", "No"]:
+        teardown()
+    else:
+        print("Temporary directory %s not removed" % utils.temporary_dir)