ci: improve FPGA file gen, fix Python black formatting

.github/scripts/build_report.py

@@ -218,8 +218,7 @@ def build(inputs, site, root=ROOT):
         fpga = {
             **{key: fpga.get(key) for key in ("schema_version", "source_sha", "source_run", "source_attempt")},
             "stages": [
-                {key: stage.get(key) for key in ("name", "exit_code", "elapsed_seconds")}
-                for stage in fpga["stages"]
+                {key: stage.get(key) for key in ("name", "exit_code", "elapsed_seconds")} for stage in fpga["stages"]
             ],
         }
     work = root / "artistic/work"
@@ -252,7 +251,7 @@ def copy(name, source):
     def archive(name, source, folders=None):
         target = site / "downloads" / name
         with zipfile.ZipFile(target, "w", zipfile.ZIP_DEFLATED) as output:
-            for directory in ([source / name for name in folders] if folders else [source]):
+            for directory in [source / name for name in folders] if folders else [source]:
                 for path in sorted(directory.rglob("*")):
                     if path.is_file():
                         output.write(path, path.relative_to(source))

.github/scripts/check_artistic.py

@@ -20,8 +20,7 @@
 ROOT = Path(__file__).resolve().parents[2]
 sys.path.insert(0, str(ROOT / "artistic" / "artistic"))
 
-from artistic.outlines import (_canvas, _def_placements, _group_placements,
-                               _lef_sizes, _options, _pixel_box)
+from artistic.outlines import _canvas, _def_placements, _group_placements, _lef_sizes, _options, _pixel_box
 from artistic.project import Project
 from artistic.presentation import presentation_geometry
 
@@ -33,34 +32,36 @@ def require(condition, message):
 
 def region(layout, layer):
     index = layout.find_layer(layer["layer"], layer["datatype"])
-    return (pya.Region(layout.top_cell().begin_shapes_rec(index))
-            if index is not None else pya.Region())
+    return pya.Region(layout.top_cell().begin_shapes_rec(index)) if index is not None else pya.Region()
 
 
 def instances(cell):
-    return collections.Counter((instance.cell.name, str(instance.cplx_trans),
-                                instance.na, instance.nb, str(instance.a), str(instance.b))
-                               for instance in cell.each_inst())
+    return collections.Counter(
+        (instance.cell.name, str(instance.cplx_trans), instance.na, instance.nb, str(instance.a), str(instance.b))
+        for instance in cell.each_inst()
+    )
 
 
 def texts(shapes):
     return collections.Counter(shape.text.to_s() for shape in shapes.each() if shape.is_text())
 
 
 def check_original_cells(base, merged):
-    layers = [(info, base.find_layer(info), merged.find_layer(info))
-              for info in base.layer_infos()]
+    layers = [(info, base.find_layer(info), merged.find_layer(info)) for info in base.layer_infos()]
     require(all(target is not None for _, _, target in layers), "Merged GDS lost an input layer")
     for cell in base.each_cell():
         target = merged.cell(cell.name)
         require(target is not None, f"Merged GDS lost cell: {cell.name}")
         require(instances(cell) == instances(target), f"Merged GDS changed instances in {cell.name}")
         for info, source_index, target_index in layers:
             source_shapes, target_shapes = cell.shapes(source_index), target.shapes(target_index)
-            require((pya.Region(source_shapes) ^ pya.Region(target_shapes)).is_empty(),
-                    f"Merged GDS changed {cell.name} on layer {info}")
-            require(texts(source_shapes) == texts(target_shapes),
-                    f"Merged GDS changed text in {cell.name} on layer {info}")
+            require(
+                (pya.Region(source_shapes) ^ pya.Region(target_shapes)).is_empty(),
+                f"Merged GDS changed {cell.name} on layer {info}",
+            )
+            require(
+                texts(source_shapes) == texts(target_shapes), f"Merged GDS changed text in {cell.name} on layer {info}"
+            )
 
 
 def main():
@@ -89,22 +90,25 @@ def main():
     check_original_cells(base, merged)
     top_design = os.environ.get("TOP_DESIGN")
     require(top_design, "Source env.sh before validating Croc artwork")
-    chip_instances = [instance for instance in base.top_cell().each_inst()
-                      if instance.cell.name == top_design]
+    chip_instances = [instance for instance in base.top_cell().each_inst() if instance.cell.name == top_design]
     require(len(chip_instances) == 1, "Expected one placed chip below the seal-ring top cell")
     transform = chip_instances[0].dcplx_trans
-    require(not transform.is_mirror() and transform.angle == 0 and transform.mag == 1,
-            "Unexpected chip rotation or scale relative to DEF")
+    require(
+        not transform.is_mirror() and transform.angle == 0 and transform.mag == 1,
+        "Unexpected chip rotation or scale relative to DEF",
+    )
     offset = settings.get("offset_um", [0, 0])
-    require(abs(transform.disp.x - offset[0]) <= base.dbu and
-            abs(transform.disp.y - offset[1]) <= base.dbu,
-            f"Outline offset {offset} does not match sealed GDS translation {transform.disp}")
+    require(
+        abs(transform.disp.x - offset[0]) <= base.dbu and abs(transform.disp.y - offset[1]) <= base.dbu,
+        f"Outline offset {offset} does not match sealed GDS translation {transform.disp}",
+    )
 
     svg = ET.parse(work / f"{project.name}_modules.svg").getroot()
     for image in svg.findall("{http://www.w3.org/2000/svg}image"):
         reference = image.get("{http://www.w3.org/1999/xlink}href")
-        require(reference and Path(reference).name == reference,
-                f"Unexpected annotated SVG image reference: {reference}")
+        require(
+            reference and Path(reference).name == reference, f"Unexpected annotated SVG image reference: {reference}"
+        )
         require((work / reference).is_file(), f"Missing annotated SVG image: {reference}")
     labels = collections.defaultdict(list)
     for text in svg.findall("{http://www.w3.org/2000/svg}text"):
@@ -118,13 +122,13 @@ def main():
         for box in groups[pattern]:
             if box[2] <= box[0] or box[3] <= box[1]:
                 continue
-            shifted = (box[0] + offset[0], box[1] + offset[1],
-                       box[2] + offset[0], box[3] + offset[1])
+            shifted = (box[0] + offset[0], box[1] + offset[1], box[2] + offset[0], box[3] + offset[1])
             pixels = _pixel_box(shifted, canvas, resolution)
             require(pixels is not None, f"Macro outside render: {pattern}")
-            require(any(pixels[0] <= x <= pixels[2] and pixels[1] <= y <= pixels[3]
-                        for x, y in labels[spec["label"]]),
-                    f"No {spec['label']} label inside placed macro {box}")
+            require(
+                any(pixels[0] <= x <= pixels[2] and pixels[1] <= y <= pixels[3] for x, y in labels[spec["label"]]),
+                f"No {spec['label']} label inside placed macro {box}",
+            )
 
     report = json.loads((work / "logo_merge.json").read_text())
     layer = report["resolved_layer"]
@@ -133,8 +137,7 @@ def main():
     expected_layer = next(item for item in generated["layers"] if item["name"] == top_metal)
     require(layer == expected_layer, "Logo is not on the technology's terminal metal")
     for key in ("feature_um", "spacing_um", "pitch_um"):
-        require(abs(report[key] - config["logo"][key]) <= base.dbu,
-                f"Logo {key} does not match the project")
+        require(abs(report[key] - config["logo"][key]) <= base.dbu, f"Logo {key} does not match the project")
     original, result, artwork = region(base, layer), region(merged, layer), region(logo, layer)
     require(not artwork.is_empty(), "Logo geometry is empty")
     require((result ^ (original + artwork)).is_empty(), "Merged metal differs from base plus logo")
@@ -143,8 +146,10 @@ def main():
     spacing = round(report["spacing_um"] / base.dbu)
     for polygon in artwork.each():
         box = polygon.bbox()
-        require(box.width() == box.height() == feature and polygon.area() == feature * feature,
-                "Logo feature is not the configured square")
+        require(
+            box.width() == box.height() == feature and polygon.area() == feature * feature,
+            "Logo feature is not the configured square",
+        )
     require(artwork.width_check(feature).is_empty(), "Logo has narrow features")
     require(artwork.space_check(spacing).is_empty(), "Logo features are too close together")
     require(original.separation_check(artwork, spacing).is_empty(), "Logo is too close to existing metal")
@@ -153,16 +158,19 @@ def main():
     metadata = json.loads((map_dir / "map.json").read_text())
     if config["map"].get("views") == ["composite", "metals"]:
         generated = json.loads((work / "map.json").read_text())
-        metals = [item["name"] for item in generated["layers"]
-                  if re.fullmatch(r"(?:Top)?Metal\d+", item["name"])]
+        metals = [item["name"] for item in generated["layers"] if re.fullmatch(r"(?:Top)?Metal\d+", item["name"])]
         require(metadata["layers"] == ["composite", *metals], "Map is missing a generated metal view")
-    require([metadata["width"], metadata["height"]] == config["map"]["resolution"],
-            "Map dimensions do not match the project")
+    require(
+        [metadata["width"], metadata["height"]] == config["map"]["resolution"],
+        "Map dimensions do not match the project",
+    )
     require((map_dir / "index.html").is_file(), "Map viewer is missing")
     tile_size, max_zoom = metadata["tile_size"], metadata["max_zoom"]
     require(tile_size == config["map"].get("tile_size", 512), "Map tile size does not match the project")
-    require(max_zoom == max(0, math.ceil(math.log2(max(config["map"]["resolution"]) / tile_size))),
-            "Map zoom pyramid does not match the project resolution")
+    require(
+        max_zoom == max(0, math.ceil(math.log2(max(config["map"]["resolution"]) / tile_size))),
+        "Map zoom pyramid does not match the project resolution",
+    )
     nx, ny = math.ceil(metadata["width"] / tile_size), math.ceil(metadata["height"] / tile_size)
     total = 0
     for name in metadata["layers"]:
@@ -179,15 +187,18 @@ def main():
         for tile in sorted(actual):
             with Image.open(layer_root / tile) as image:
                 image.load()
-                require(image.size == (tile_size, tile_size) and image.mode == "RGBA",
-                        f"Invalid map tile: {name}/{tile}")
+                require(
+                    image.size == (tile_size, tile_size) and image.mode == "RGBA", f"Invalid map tile: {name}/{tile}"
+                )
                 red, green, blue, alpha = image.split()
                 nonblank |= any(low != high for low, high in image.getextrema()[:3])
                 if name != "composite" and config["map"].get("layer_style", "mask") == "mask":
                     require(alpha.getextrema() == (255, 255), f"Transparent mask: {name}/{tile}")
-                    require(ImageChops.difference(red, green).getbbox() is None and
-                            ImageChops.difference(red, blue).getbbox() is None,
-                            f"Colored mask: {name}/{tile}")
+                    require(
+                        ImageChops.difference(red, green).getbbox() is None
+                        and ImageChops.difference(red, blue).getbbox() is None,
+                        f"Colored mask: {name}/{tile}",
+                    )
         require(nonblank, f"Map layer is blank: {name}")
         total += len(actual)
     print(f"[INFO][Artistic] Validated render, all module labels, sealed alignment, logo spacing and {total} map tiles")

.github/scripts/record_ci.py

@@ -16,7 +16,6 @@
 import time
 from typing import BinaryIO, Dict, Optional, Sequence
 
-
 ROOT = Path(__file__).resolve().parents[2]
 ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
 

.github/scripts/report_metrics.py

@@ -14,7 +14,6 @@
 
 import odb
 
-
 REPO_ROOT = Path(__file__).resolve().parents[2]
 NUMBER = r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?"
 

.github/scripts/test_build_report.py

@@ -198,7 +198,7 @@ def test_fpga_pass_links_to_the_public_run_without_logs(self):
         self.assertIn('class="status pass">Pass', node)
         self.assertIn("12m34s", node)
         self.assertIn("Commit aaaaaaa", node)
-        self.assertIn('>FPGA</a>', html)
+        self.assertIn(">FPGA</a>", html)
         self.assertFalse((self.root / "site/logs/fpga.html").exists())
 
     def test_fpga_artifact_extra_fields_are_not_published(self):
@@ -211,7 +211,9 @@ def test_fpga_artifact_extra_fields_are_not_published(self):
         published = (self.root / "site/data.json").read_text()
         for leaked in ("gitlab.example", "runner-1", "secret.log", "fpga-host"):
             self.assertNotIn(leaked, published)
-        self.assertEqual(json.loads(published)["fpga"]["stages"], [{"name": "fpga", "exit_code": 0, "elapsed_seconds": 754}])
+        self.assertEqual(
+            json.loads(published)["fpga"]["stages"], [{"name": "fpga", "exit_code": 0, "elapsed_seconds": 754}]
+        )
 
     def test_failed_fpga_is_shown_and_does_not_fail_the_build(self):
         self.add_fpga(exit_code=1)

.github/scripts/test_record_ci.py

@@ -11,7 +11,6 @@
 import unittest
 from unittest.mock import patch
 
-
 SCRIPT = Path(__file__).with_name("record_ci.py")
 SPEC = importlib.util.spec_from_file_location("record_ci", SCRIPT)
 record_ci = importlib.util.module_from_spec(SPEC)
@@ -89,7 +88,9 @@ def test_result_records_outcome_of_a_step_without_a_command(self):
             with self.subTest(outcome=outcome):
                 self.assertEqual(record_ci.record_result("fpga", "fpga", outcome, 12.3456, self.root), exit_code)
                 data = json.loads((self.root / "ci-results/fpga.json").read_text())
-                self.assertEqual(data["stages"], [{"name": "fpga", "elapsed_seconds": 12.346, "exit_code": exit_code, "log": None}])
+                self.assertEqual(
+                    data["stages"], [{"name": "fpga", "elapsed_seconds": 12.346, "exit_code": exit_code, "log": None}]
+                )
                 self.assertEqual(data["tests"], [])
                 self.assertEqual(
                     (data["schema_version"], data["source_sha"], data["source_run"], data["source_attempt"]),

.github/scripts/test_report_metrics.py

@@ -8,7 +8,6 @@
 from pathlib import Path
 import unittest
 
-
 spec = importlib.util.spec_from_file_location("report_metrics", Path(__file__).with_name("report_metrics.py"))
 metrics = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(metrics)

.gitignore

@@ -13,6 +13,8 @@
 /tmp
 /calibre
 /docs
+__pycache__/
+*.pyc
 /technology
 /.setPDK.csh
 # public PDK mirror

klayout/scripts/def2stream.py

@@ -11,7 +11,7 @@
 tech = pya.Technology()
 layoutOptions = tech.load_layout_options
 layoutOptions.lefdef_config.map_file = layer_map
-layoutOptions.lefdef_config.lef_files = lef_files.split(' ')
+layoutOptions.lefdef_config.lef_files = lef_files.split(" ")
 
 # Load def file
 main_layout = pya.Layout()
@@ -20,7 +20,7 @@
 
 print("[INFO] Reporting cells after loading DEF ...")
 for i in main_layout.each_cell():
- print("[INFO] '{0}'".format(i.name))
+    print("[INFO] '{0}'".format(i.name))
 
 # Clear cells
 top_cell_index = main_layout.cell(design_name).cell_index()
@@ -29,15 +29,15 @@
 #  - KLayout is prepending VIA_ when reading DEF that instantiates LEF's via
 print("[INFO] Clearing cells...")
 for i in main_layout.each_cell():
-  if i.cell_index() != top_cell_index:
-    if not i.name.startswith("VIA_"):
-      i.clear()
+    if i.cell_index() != top_cell_index:
+        if not i.name.startswith("VIA_"):
+            i.clear()
 
 # Load in the gds to merge
 print("[INFO] Merging GDS/OAS files...")
 for fil in gds_files.split():
-  print("\t{0}".format(fil))
-  main_layout.read(fil)
+    print("\t{0}".format(fil))
+    main_layout.read(fil)
 
 # Copy the top level only to a new layout
 print("[INFO] Copying toplevel cell '{0}'".format(design_name))
@@ -49,31 +49,30 @@
 print("[INFO] Checking for missing cell from GDS/OAS...")
 missing_cell = False
 regex = None
-if gds_allow_empty == 'True':
+if gds_allow_empty == "True":
     print("[INFO] Allow GDS_ALLOW_EMPTY.")
 for i in top_only_layout.each_cell():
-  if i.is_empty():
-    missing_cell = True
-    if gds_allow_empty == 'True':
-        print("[WARNING] LEF Cell '{0}' ignored. Matches GDS_ALLOW_EMPTY.".format(i.name))
-    else:
-        print("[ERROR] LEF Cell '{0}' has no matching GDS/OAS cell."
-              " Cell will be empty.".format(i.name))
-        errors += 1
+    if i.is_empty():
+        missing_cell = True
+        if gds_allow_empty == "True":
+            print("[WARNING] LEF Cell '{0}' ignored. Matches GDS_ALLOW_EMPTY.".format(i.name))
+        else:
+            print("[ERROR] LEF Cell '{0}' has no matching GDS/OAS cell." " Cell will be empty.".format(i.name))
+            errors += 1
 
 if not missing_cell:
-  print("[INFO] All LEF cells have matching GDS/OAS cells")
+    print("[INFO] All LEF cells have matching GDS/OAS cells")
 
 print("[INFO] Checking for orphan cell in the final layout...")
 orphan_cell = False
 for i in top_only_layout.each_cell():
-  if i.name != design_name and i.parent_cells() == 0:
-    orphan_cell = True
-    print("[ERROR] Found orphan cell '{0}'".format(i.name))
-    errors += 1
+    if i.name != design_name and i.parent_cells() == 0:
+        orphan_cell = True
+        print("[ERROR] Found orphan cell '{0}'".format(i.name))
+        errors += 1
 
 if not orphan_cell:
-  print("[INFO] No orphan cells")
+    print("[INFO] No orphan cells")
 
 
 # Write out the GDS

klayout/scripts/generate_seal_ring.py

@@ -9,14 +9,11 @@
 
 import pya
 
-
 croc_root = os.environ["CROC_ROOT"]
 klayout_path = os.environ["KLAYOUT_PATH"]
 pdk = os.environ["CROC_PDK"]
 klayout_python = os.path.join(klayout_path, "python")
-shared_klayout_python = os.path.join(
-    croc_root, "ihp13", "sg13g2", "ihp-sg13g2", "libs.tech", "klayout", "python"
-)
+shared_klayout_python = os.path.join(croc_root, "ihp13", "sg13g2", "ihp-sg13g2", "libs.tech", "klayout", "python")
 sys.path.extend(
     [
         klayout_python,
@@ -27,7 +24,6 @@
 from cni.dlo import PyCellContext
 from cni.tech import Tech
 
-
 if pdk == "sg13cmos5l":
     pcell_package = "sg13cmos5l_pycell_lib"
 elif pdk == "sg13g2":
@@ -55,9 +51,7 @@ def load_module(name, path):
 
 
 load_module(f"{package_name}.sg13_tech", os.path.join(pcell_root, "sg13_tech.py"))
-sealring_module = load_module(
-    f"{package_name}.ihp.sealring_code", os.path.join(ihp_root, "sealring_code.py")
-)
+sealring_module = load_module(f"{package_name}.ihp.sealring_code", os.path.join(ihp_root, "sealring_code.py"))
 
 tech = Tech.get("SG13_dev")
 Tech.techInUse = tech.getTechParams()["libName"]