fix --fields list leak, reject non-dict JSON input, add --semantic-diff mode

--fields incorrectly preserved leaf lists (e.g. regions, tags) whose keys
were not in the field set, defeating the purpose of token-reduction for
agent pipelines. Now only recurses into structural containers (dicts and
lists-of-dicts for block hierarchy).

jsontohcl2 --dry-run and normal mode silently accepted non-dict JSON
(arrays, scalars) producing garbled output with exit 0. load_python()
now raises TypeError, caught by the existing handler to exit 2.

New --semantic-diff ORIGINAL mode compares parsed HCL dict against JSON
dict using diff_dicts(), completely ignoring formatting differences
(alignment, trailing commas, comments, array layout). --diff-json flag
outputs structured JSON for machine consumption.

Documented --fragment inner-quote string convention in CLI help, epilog,
and docs to prevent the common first-use mistake of passing plain JSON
strings.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: kkozik-amplify

CLAUDE.md

@@ -92,7 +92,7 @@ All CLIs use structured error output (plain text to stderr) and distinct exit co
 | 2 | All unparsable | Bad HCL structure | Parse error |
 | 3 | — | — | Query error |
 | 4 | I/O error | I/O error | I/O error |
-| 5 | — | Differences found (`--diff`) | — |
+| 5 | — | Differences found (`--diff` / `--semantic-diff`) | — |
 
 ### `hcl2tojson`
 
@@ -114,14 +114,16 @@ Key flags: `--ndjson`, `--compact`, `--only`/`--exclude`, `--fields`, `-q`/`--qu
 ### `jsontohcl2`
 
 ```
-jsontohcl2 file.json                          # single file to stdout
-jsontohcl2 --diff original.tf modified.json  # preview changes
-jsontohcl2 --dry-run file.json               # convert without writing
-jsontohcl2 --fragment -                       # attribute snippets from stdin
+jsontohcl2 file.json                                       # single file to stdout
+jsontohcl2 --diff original.tf modified.json                # preview text changes
+jsontohcl2 --semantic-diff original.tf modified.json       # semantic-only changes
+jsontohcl2 --semantic-diff original.tf --diff-json m.json  # semantic diff as JSON
+jsontohcl2 --dry-run file.json                             # convert without writing
+jsontohcl2 --fragment -                                    # attribute snippets from stdin
 jsontohcl2 --indent 4 --no-align file.json
 ```
 
-Key flags: `--diff ORIGINAL`, `--dry-run`, `--fragment`, `-q`/`--quiet`, `--indent N`, `--no-align`, `--colon-separator`.
+Key flags: `--diff ORIGINAL`, `--semantic-diff ORIGINAL`, `--diff-json`, `--dry-run`, `--fragment`, `-q`/`--quiet`, `--indent N`, `--no-align`, `--colon-separator`.
 
 Add new options as `parser.add_argument()` calls in the relevant entry point module.
 

cli/hcl_to_json.py

@@ -9,7 +9,7 @@
 from hcl2 import load
 from hcl2.utils import SerializationOptions
 from hcl2.version import __version__
-from .helpers import (
+from cli.helpers import (
     EXIT_IO_ERROR,
     EXIT_PARSE_ERROR,
     EXIT_PARTIAL,
@@ -58,11 +58,15 @@ def _project_fields(data, field_set):
         for key, val in data.items():
             if key in field_set or key.startswith("__"):
                 result[key] = val
-            elif isinstance(val, (dict, list)):
+            elif isinstance(val, dict):
                 projected = _project_fields(val, field_set)
                 if projected:
                     result[key] = projected
-            # else: leaf value not in field_set — drop it
+            elif isinstance(val, list) and any(isinstance(item, dict) for item in val):
+                projected = _project_fields(val, field_set)
+                if projected:
+                    result[key] = projected
+            # else: leaf value (scalar or leaf list) not in field_set — drop it
         return result
     if isinstance(data, list):
         out = [_project_fields(item, field_set) for item in data]
@@ -453,3 +457,7 @@ def _resolve_file_paths(paths: List[str], parser) -> List[str]:
     if not file_paths:
         parser.error("no HCL files found in the given paths")
     return file_paths
+
+
+if __name__ == "__main__":
+    main()

cli/json_to_hcl.py

@@ -8,11 +8,14 @@
 from io import StringIO
 from typing import TextIO
 
+import hcl2
 from hcl2 import dump
 from hcl2.deserializer import DeserializerOptions
 from hcl2.formatter import FormatterOptions
+from hcl2.query.diff import diff_dicts, format_diff_json, format_diff_text
+from hcl2.utils import SerializationOptions
 from hcl2.version import __version__
-from .helpers import (
+from cli.helpers import (
     EXIT_DIFF,
     EXIT_IO_ERROR,
     EXIT_PARSE_ERROR,
@@ -81,19 +84,26 @@ def _strip_block_markers(data):
 
 _EXAMPLES = """\
 examples:
-  jsontohcl2 file.json                          # single file to stdout
-  jsontohcl2 a.json b.json -o out/             # multiple files to output dir
-  jsontohcl2 --diff original.tf modified.json  # preview changes
-  jsontohcl2 --dry-run file.json               # convert without writing
-  jsontohcl2 --fragment -                       # attribute snippet from stdin
-  echo '{"x": 1}' | jsontohcl2                 # stdin (no args needed)
+  jsontohcl2 file.json                                   # single file to stdout
+  jsontohcl2 a.json b.json -o out/                      # multiple files to output dir
+  jsontohcl2 --diff original.tf modified.json            # preview text changes
+  jsontohcl2 --semantic-diff original.tf modified.json   # semantic-only changes
+  jsontohcl2 --semantic-diff original.tf --diff-json m.json  # semantic diff as JSON
+  jsontohcl2 --dry-run file.json                         # convert without writing
+  jsontohcl2 --fragment -                                # attribute snippet from stdin
+  echo '{"x": 1}' | jsontohcl2                          # stdin (no args needed)
+
+fragment string format:
+  Strings use python-hcl2's inner-quote convention. To produce HCL "value",
+  the JSON string must be: "\\"value\\"". Unquoted strings become identifiers.
+  Example: {"name": "\\"test\\"", "count": 3}  =>  name = "test"  count = 3
 
 exit codes:
   0  Success
   1  JSON parse error
   2  Valid JSON but incompatible HCL structure
   4  I/O error (file not found)
-  5  Differences found (--diff mode only)
+  5  Differences found (--diff / --semantic-diff)
 """
 
 
@@ -136,10 +146,22 @@ def main():  # pylint: disable=too-many-branches,too-many-statements,too-many-lo
         action="store_true",
         help="Convert and print to stdout without writing files",
     )
+    mode_group.add_argument(
+        "--semantic-diff",
+        metavar="ORIGINAL",
+        help="Show semantic-only diff against ORIGINAL (ignores formatting)",
+    )
     mode_group.add_argument(
         "--fragment",
         action="store_true",
-        help="Treat input as a JSON fragment (attribute dict, not full HCL document)",
+        help="Treat input as a JSON fragment (attribute dict, not full HCL "
+        'document). Strings must use inner quotes: \'"\\"value\\""\' for HCL '
+        '"value", bare strings become identifiers',
+    )
+    parser.add_argument(
+        "--diff-json",
+        action="store_true",
+        help="Output diff results as JSON (works with --diff and --semantic-diff)",
     )
     parser.add_argument("--version", action="version", version=__version__)
 
@@ -262,6 +284,47 @@ def convert(in_file, out_file):
                 sys.exit(EXIT_DIFF)
             return
 
+        # --semantic-diff mode: compare semantic dicts (ignores formatting)
+        if args.semantic_diff:
+            if len(paths) != 1:
+                parser.error("--semantic-diff requires exactly one input file")
+            json_path = paths[0]
+            original_path = args.semantic_diff
+
+            if not os.path.isfile(original_path):
+                print(
+                    _error(
+                        f"File not found: {original_path}",
+                        error_type="io_error",
+                        file=original_path,
+                    ),
+                    file=sys.stderr,
+                )
+                sys.exit(EXIT_IO_ERROR)
+
+            # Parse original HCL → normalized dict
+            sem_opts = SerializationOptions(
+                with_comments=False, with_meta=False, explicit_blocks=True
+            )
+            with open(original_path, "r", encoding="utf-8") as f:
+                dict_original = hcl2.load(f, serialization_options=sem_opts)
+
+            # Load modified JSON → dict
+            if json_path == "-":
+                dict_modified = json.load(sys.stdin)
+            else:
+                with open(json_path, "r", encoding="utf-8") as f:
+                    dict_modified = json.load(f)
+
+            entries = diff_dicts(dict_original, dict_modified)
+            if entries:
+                if args.diff_json:
+                    print(format_diff_json(entries))
+                else:
+                    print(format_diff_text(entries))
+                sys.exit(EXIT_DIFF)
+            return
+
         # --dry-run mode: convert to stdout without writing
         if args.dry_run:
             if len(paths) != 1:
@@ -381,3 +444,7 @@ def convert(in_file, out_file):
             file=sys.stderr,
         )
         sys.exit(EXIT_IO_ERROR)
+
+
+if __name__ == "__main__":
+    main()

docs/01_getting_started.md

@@ -201,16 +201,18 @@ echo 'x = 1' | hcl2tojson                  # stdin (no args needed)
 Convert JSON files to HCL2. Accepts files, directories, glob patterns, or stdin (default when no args given).
 
 ```sh
-jsontohcl2 output.json                       # single file to stdout
-jsontohcl2 output.json -o main.tf            # single file to output file
-jsontohcl2 output/ -o terraform/             # directory conversion
-jsontohcl2 --diff original.tf modified.json  # preview changes as unified diff
-jsontohcl2 --dry-run file.json               # convert without writing
-jsontohcl2 --fragment -                       # attribute snippets from stdin
-echo '{"x": 1}' | jsontohcl2                 # stdin (no args needed)
+jsontohcl2 output.json                                    # single file to stdout
+jsontohcl2 output.json -o main.tf                        # single file to output file
+jsontohcl2 output/ -o terraform/                         # directory conversion
+jsontohcl2 --diff original.tf modified.json              # preview changes as unified diff
+jsontohcl2 --semantic-diff original.tf modified.json     # semantic-only diff (ignores formatting)
+jsontohcl2 --semantic-diff original.tf --diff-json m.json  # semantic diff as JSON
+jsontohcl2 --dry-run file.json                           # convert without writing
+jsontohcl2 --fragment -                                  # attribute snippets from stdin
+echo '{"x": 1}' | jsontohcl2                            # stdin (no args needed)
 ```
 
-**Exit codes:** 0 = success, 1 = JSON parse error, 2 = bad HCL structure, 4 = I/O error, 5 = differences found (`--diff`).
+**Exit codes:** 0 = success, 1 = JSON parse error, 2 = bad HCL structure, 4 = I/O error, 5 = differences found (`--diff` / `--semantic-diff`).
 
 **Flags:**
 
@@ -220,8 +222,10 @@ echo '{"x": 1}' | jsontohcl2                 # stdin (no args needed)
 | `-s` | Skip un-parsable files |
 | `-q`, `--quiet` | Suppress progress output on stderr |
 | `--diff ORIGINAL` | Show unified diff against ORIGINAL file (exit 0 = identical, 5 = differs) |
+| `--semantic-diff ORIGINAL` | Show semantic-only diff against ORIGINAL (ignores formatting differences) |
+| `--diff-json` | Output diff results as JSON (works with `--diff` and `--semantic-diff`) |
 | `--dry-run` | Convert and print to stdout without writing files |
-| `--fragment` | Treat input as attribute dict, not full HCL document |
+| `--fragment` | Treat input as attribute dict, not full HCL document (see note below) |
 | `--indent N` | Indentation width (default: 2) |
 | `--colon-separator` | Use `:` instead of `=` in object elements |
 | `--no-trailing-comma` | Omit trailing commas in object elements |
@@ -233,6 +237,8 @@ echo '{"x": 1}' | jsontohcl2                 # stdin (no args needed)
 | `--no-align` | Disable vertical alignment of attributes and object elements |
 | `--version` | Show version and exit |
 
+> **Note on `--fragment` string format:** `--fragment` uses python-hcl2's standard JSON format, where HCL string values carry inner quotes. To produce the HCL attribute `name = "test"`, the JSON value must be `"\"test\""` (escaped inner quotes). A plain JSON string like `"test"` becomes the bare identifier `test`. This is the same convention used by `hcl2tojson` output — so piping `hcl2tojson` output into `jsontohcl2 --fragment` works correctly. Numbers, booleans, and expressions (`var.foo`, `local.name`) do not need quoting.
+
 ### hq
 
 Query HCL2 files by structure, with optional Python expressions.

hcl2/deserializer.py

@@ -101,13 +101,13 @@ def _transformer(self) -> RuleTransformer:
 
     def load_python(self, value: Any) -> StartRule:
         """Deserialize a Python object into a StartRule tree."""
-        if isinstance(value, dict):
-            # Top-level dict is always a body (attributes + blocks), not an object
-            children = self._deserialize_block_elements(value)
-            result = StartRule([BodyRule(children)])
-        else:
-            result = StartRule([self._deserialize(value)])
-        return result
+        if not isinstance(value, dict):
+            raise TypeError(
+                f"Expected dict for top-level HCL body, got {type(value).__name__}"
+            )
+        # Top-level dict is always a body (attributes + blocks), not an object
+        children = self._deserialize_block_elements(value)
+        return StartRule([BodyRule(children)])
 
     def loads(self, value: str) -> LarkElement:
         """Deserialize a JSON string into a LarkElement tree."""