Add json data type

CHANGELOG.md

@@ -1,5 +1,11 @@
 ## Release notes
 
+### 0.14.0 -- TBA
+* Add `json` data type ([#245](https://github.com/datajoint/datajoint-python/issues/245)) PR [#1051](https://github.com/datajoint/datajoint-python/pull/1051)
+- Fix lingering prints by replacing with logs PR [#1051](https://github.com/datajoint/datajoint-python/pull/1051)
+- `table.progress()` defaults to no stdout PR [#1051](https://github.com/datajoint/datajoint-python/pull/1051)
+- `table.describe()` defaults to no stdout PR [#1051](https://github.com/datajoint/datajoint-python/pull/1051)
+
 ### 0.13.8 -- Sep 21, 2022
 * Add - New documentation structure based on markdown PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)
 * Bugfix - Fix queries with backslashes ([#999](https://github.com/datajoint/datajoint-python/issues/999)) PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)

LNX-docker-compose.yml

@@ -13,6 +13,11 @@ services:
     #   - "3306:3306"
     # volumes:
     #   - ./mysql/data:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
+      timeout: 30s
+      retries: 5
+      interval: 15s
   minio:
     <<: *net
     image: minio/minio:$MINIO_VER
@@ -27,9 +32,9 @@ services:
     command: server --address ":9000" /data
     healthcheck:
       test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
-      timeout: 5s
-      retries: 60
-      interval: 1s
+      timeout: 30s
+      retries: 5
+      interval: 15s
   fakeservices.datajoint.io:
     <<: *net
     image: datajoint/nginx:v0.2.3

datajoint/admin.py

@@ -3,6 +3,9 @@
 from .connection import conn
 from .settings import config
 from .utils import user_choice
+import logging
+
+logger = logging.getLogger(__name__.split(".")[0])
 
 
 def set_password(
@@ -13,10 +16,10 @@ def set_password(
         new_password = getpass("New password: ")
         confirm_password = getpass("Confirm password: ")
         if new_password != confirm_password:
-            print("Failed to confirm the password! Aborting password change.")
+            logger.warn("Failed to confirm the password! Aborting password change.")
             return
     connection.query("SET PASSWORD = PASSWORD('%s')" % new_password)
-    print("Password updated.")
+    logger.info("Password updated.")
 
     if update_config or (
         update_config is None and user_choice("Update local setting?") == "yes"
@@ -81,7 +84,7 @@ def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
                 try:
                     connection.query("kill %d" % pid)
                 except pymysql.err.InternalError:
-                    print("Process not found")
+                    logger.warn("Process not found")
 
 
 def kill_quick(restriction=None, connection=None):

datajoint/autopopulate.py

@@ -314,7 +314,7 @@ def _populate1(
                 finally:
                     self.__class__._allow_insert = False
 
-    def progress(self, *restrictions, display=True):
+    def progress(self, *restrictions, display=False):
         """
         Report the progress of populating the table.
         :return: (remaining, total) -- numbers of tuples to be populated
@@ -323,9 +323,9 @@ def progress(self, *restrictions, display=True):
         total = len(todo)
         remaining = len(todo - self.target)
         if display:
-            print(
-                "%-20s" % self.__class__.__name__,
-                "Completed %d of %d (%2.1f%%)   %s"
+            logger.info(
+                "%-20s" % self.__class__.__name__
+                + " Completed %d of %d (%2.1f%%)   %s"
                 % (
                     total - remaining,
                     total,
@@ -334,6 +334,5 @@ def progress(self, *restrictions, display=True):
                         datetime.datetime.now(), "%Y-%m-%d %H:%M:%S"
                     ),
                 ),
-                flush=True,
             )
         return remaining, total

datajoint/condition.py

@@ -8,8 +8,29 @@
 import decimal
 import numpy
 import pandas
+import json
 from .errors import DataJointError
 
+JSON_PATTERN = re.compile(
+    r"^(?P<attr>\w+)(\.(?P<path>[\w.*\[\]]+))?(:(?P<type>[\w(,\s)]+))?$"
+)
+
+
+def translate_attribute(key):
+    match = JSON_PATTERN.match(key)
+    if match is None:
+        return match, key
+    match = match.groupdict()
+    if match["path"] is None:
+        return match, match["attr"]
+    else:
+        return match, "json_value(`{}`, _utf8mb4'$.{}'{})".format(
+            *{
+                k: ((f" returning {v}" if k == "type" else v) if v else "")
+                for k, v in match.items()
+            }.values()
+        )
+
 
 class PromiscuousOperand:
     """
@@ -94,35 +115,58 @@ def make_condition(query_expression, condition, columns):
     from .expression import QueryExpression, Aggregation, U
 
     def prep_value(k, v):
-        """prepare value v for inclusion as a string in an SQL condition"""
-        if query_expression.heading[k].uuid:
+        """prepare SQL condition"""
+        key_match, k = translate_attribute(k)
+        if key_match["path"] is None:
+            k = f"`{k}`"
+        if (
+            query_expression.heading[key_match["attr"]].json
+            and key_match["path"] is not None
+            and isinstance(v, dict)
+        ):
+            return f"{k}='{json.dumps(v)}'"
+        if v is None:
+            return f"{k} IS NULL"
+        if query_expression.heading[key_match["attr"]].uuid:
             if not isinstance(v, uuid.UUID):
                 try:
                     v = uuid.UUID(v)
                 except (AttributeError, ValueError):
                     raise DataJointError(
                         "Badly formed UUID {v} in restriction by `{k}`".format(k=k, v=v)
                     )
-            return "X'%s'" % v.bytes.hex()
+            return f"{k}=X'{v.bytes.hex()}'"
         if isinstance(
-            v, (datetime.date, datetime.datetime, datetime.time, decimal.Decimal)
+            v,
+            (
+                datetime.date,
+                datetime.datetime,
+                datetime.time,
+                decimal.Decimal,
+                list,
+            ),
         ):
-            return '"%s"' % v
+            return f'{k}="{v}"'
         if isinstance(v, str):
-            return '"%s"' % v.replace("%", "%%").replace("\\", "\\\\")
-        return "%r" % v
+            v = v.replace("%", "%%").replace("\\", "\\\\")
+            return f'{k}="{v}"'
+        return f"{k}={v}"
+
+    def join_conditions(negate, restrictions, operator="AND"):
+        return ("NOT (%s)" if negate else "%s") % (
+            f"({f') {operator} ('.join(restrictions)})"
+        )
 
     negate = False
     while isinstance(condition, Not):
         negate = not negate
         condition = condition.restriction
-    template = "NOT (%s)" if negate else "%s"
 
     # restrict by string
     if isinstance(condition, str):
         columns.update(extract_column_names(condition))
-        return template % condition.strip().replace(
-            "%", "%%"
+        return join_conditions(
+            negate, restrictions=[condition.strip().replace("%", "%%")]
         )  # escape %, see issue #376
 
     # restrict by AndList
@@ -139,7 +183,7 @@ def prep_value(k, v):
             return negate  # if any item is False, the whole thing is False
         if not items:
             return not negate  # and empty AndList is True
-        return template % ("(" + ") AND (".join(items) + ")")
+        return join_conditions(negate, restrictions=items)
 
     # restriction by dj.U evaluates to True
     if isinstance(condition, U):
@@ -151,23 +195,19 @@ def prep_value(k, v):
 
     # restrict by a mapping/dict -- convert to an AndList of string equality conditions
     if isinstance(condition, collections.abc.Mapping):
-        common_attributes = set(condition).intersection(query_expression.heading.names)
+        common_attributes = set(c.split(".", 1)[0] for c in condition).intersection(
+            query_expression.heading.names
+        )
         if not common_attributes:
             return not negate  # no matching attributes -> evaluates to True
         columns.update(common_attributes)
-        return template % (
-            "("
-            + ") AND (".join(
-                "`%s`%s"
-                % (
-                    k,
-                    " IS NULL"
-                    if condition[k] is None
-                    else f"={prep_value(k, condition[k])}",
-                )
-                for k in common_attributes
-            )
-            + ")"
+        return join_conditions(
+            negate,
+            restrictions=[
+                prep_value(k, v)
+                for k, v in condition.items()
+                if k.split(".", 1)[0] in common_attributes
+            ],
         )
 
     # restrict by a numpy record -- convert to an AndList of string equality conditions
@@ -178,12 +218,9 @@ def prep_value(k, v):
         if not common_attributes:
             return not negate  # no matching attributes -> evaluate to True
         columns.update(common_attributes)
-        return template % (
-            "("
-            + ") AND (".join(
-                "`%s`=%s" % (k, prep_value(k, condition[k])) for k in common_attributes
-            )
-            + ")"
+        return join_conditions(
+            negate,
+            restrictions=[prep_value(k, condition[k]) for k in common_attributes],
         )
 
     # restrict by a QueryExpression subclass -- trigger instantiation and move on
@@ -231,7 +268,11 @@ def prep_value(k, v):
         ]  # ignore False conditions
         if any(item is True for item in or_list):  # if any item is True, entirely True
             return not negate
-        return template % ("(%s)" % " OR ".join(or_list)) if or_list else negate
+        return (
+            join_conditions(negate, restrictions=or_list, operator="OR")
+            if or_list
+            else negate
+        )
 
 
 def extract_column_names(sql_expression):

datajoint/declare.py

@@ -7,6 +7,7 @@
 import logging
 from .errors import DataJointError, _support_filepath_types, FILEPATH_FEATURE_SWITCH
 from .attribute_adapter import get_adapter
+from .condition import translate_attribute
 
 UUID_DATA_TYPE = "binary(16)"
 MAX_TABLE_NAME_LENGTH = 64
@@ -23,6 +24,7 @@
         DECIMAL=r"(decimal|numeric)(\s*\(.+\))?(\s+unsigned)?$",
         FLOAT=r"(double|float|real)(\s*\(.+\))?(\s+unsigned)?$",
         STRING=r"(var)?char\s*\(.+\)$",
+        JSON=r"json$",
         ENUM=r"enum\s*\(.+\)$",
         BOOL=r"bool(ean)?$",  # aliased to tinyint(1)
         TEMPORAL=r"(date|datetime|time|timestamp|year)(\s*\(.+\))?$",
@@ -129,25 +131,9 @@ def build_attribute_parser():
     return attribute_name + pp.Optional(default) + colon + data_type + comment
 
 
-def build_index_parser():
-    left = pp.Literal("(").suppress()
-    right = pp.Literal(")").suppress()
-    unique = pp.Optional(pp.CaselessKeyword("unique")).setResultsName("unique")
-    index = pp.CaselessKeyword("index").suppress()
-    attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]"))
-    return (
-        unique
-        + index
-        + left
-        + pp.delimitedList(attribute_name).setResultsName("attr_list")
-        + right
-    )
-
-
 foreign_key_parser_old = build_foreign_key_parser_old()
 foreign_key_parser = build_foreign_key_parser()
 attribute_parser = build_attribute_parser()
-index_parser = build_index_parser()
 
 
 def is_foreign_key(line):
@@ -341,7 +327,7 @@ def prepare_declare(definition, context):
                 foreign_key_sql,
                 index_sql,
             )
-        elif re.match(r"^(unique\s+)?index[^:]*$", line, re.I):  # index
+        elif re.match(r"^(unique\s+)?index\s*.*$", line, re.I):  # index
             compile_index(line, index_sql)
         else:
             name, sql, store = compile_attribute(line, in_key, foreign_key_sql, context)
@@ -515,10 +501,23 @@ def alter(definition, old_definition, context):
 
 
 def compile_index(line, index_sql):
-    match = index_parser.parseString(line)
+    def format_attribute(attr):
+        match, attr = translate_attribute(attr)
+        if match is None:
+            return attr
+        elif match["path"] is None:
+            return f"`{attr}`"
+        else:
+            return f"({attr})"
+
+    match = re.match(
+        r"(?P<unique>unique\s+)?index\s*\(\s*(?P<args>.*)\)", line, re.I
+    ).groupdict()
+    attr_list = re.findall(r"(?:[^,(]|\([^)]*\))+", match["args"])
     index_sql.append(
-        "{unique} index ({attrs})".format(
-            unique=match.unique, attrs=",".join("`%s`" % a for a in match.attr_list)
+        "{unique}index ({attrs})".format(
+            unique="unique " if match["unique"] else "",
+            attrs=",".join(format_attribute(a.strip()) for a in attr_list),
         )
     )
 

datajoint/diagram.py

@@ -385,9 +385,7 @@ def make_dot(self):
                 if name.split(".")[0] in self.context:
                     cls = eval(name, self.context)
                     assert issubclass(cls, Table)
-                    description = (
-                        cls().describe(context=self.context, printout=False).split("\n")
-                    )
+                    description = cls().describe(context=self.context).split("\n")
                     description = (
                         "-" * 30
                         if q.startswith("---")

datajoint/expression.py

@@ -14,6 +14,7 @@
     assert_join_compatibility,
     extract_column_names,
     PromiscuousOperand,
+    translate_attribute,
 )
 from .declare import CONSTANT_LITERALS
 
@@ -342,6 +343,9 @@ def proj(self, *attributes, **named_attributes):
         from other attributes available before the projection.
         Each attribute name can only be used once.
         """
+        named_attributes = {
+            k: translate_attribute(v)[1] for k, v in named_attributes.items()
+        }
         # new attributes in parentheses are included again with the new name without removing original
         duplication_pattern = re.compile(
             rf'^\s*\(\s*(?!{"|".join(CONSTANT_LITERALS)})(?P<name>[a-zA-Z_]\w*)\s*\)\s*$'