Further improves deployment information, makes package info really useful

build/microzig-package.json

@@ -1,5 +1,5 @@
 {
-    "package_name": "build",
+    "package_name": "microzig-build",
     "package_type": "build",
     "external_dependencies": {
         "uf2": {

core/microzig-package.json

@@ -1,5 +1,5 @@
 {
-    "package_name": "core",
+    "package_name": "microzig-core",
     "package_type": "core",
     "external_dependencies": {
         "umm-zig": {

tools/bundle.py

@@ -14,7 +14,7 @@ from enum import Enum as StrEnum
 import pathspec
 import stat 
 from marshmallow import fields as mm_fields
-from typing import Optional
+from typing import Optional, Any
 
 
 VERBOSE = False 
@@ -29,6 +29,7 @@ REQUIRED_TOOLS = [
     # "dirname",
     # "realpath",
 ]
+DEPLOYMENT_BASE="https://download.microzig.tech/packages"
 
 REPO_ROOT = Path(__file__).parent.parent
 assert REPO_ROOT.is_dir()
@@ -44,7 +45,7 @@ class PackageType(StrEnum):
 @dataclass_json
 @dataclass
 class Archive:
-    size: int
+    size: str
     sha256sum: str
 
 @dataclass_json
@@ -80,9 +81,30 @@ class PackageConfiguration:
     created: Optional[Timestamp] = field(default = None)
     package: Optional[Package]  = field(default= None)
 
+    download_url: Optional[str] = field(default=None)
+
+    microzig: Optional[Any] = field(default=None)
+
     # inner fields:
     # package_dir: Path = field(default=None, metadata = dcj_config(exclude=JsonExclude.ALWAYS))
 
+@dataclass_json
+@dataclass
+class PackageDesc:
+    name: str
+    type: PackageType
+    version: str # semver
+    metadata: str # url to json
+    download: str # url to tar.gz
+
+@dataclass_json
+@dataclass
+class PackageIndex:
+    
+    last_update: Timestamp
+    packages: list[PackageDesc]
+
+PackageIndexSchema = PackageIndex.schema()
 PackageSchema = Package.schema()
 PackageConfigurationSchema = PackageConfiguration.schema()
 
@@ -261,8 +283,6 @@ def main():
         pathspec.patterns.GitWildMatchPattern("microzig-package.json")
     )
 
-    print(global_ignore_spec)      
-
     # Fetch and find all packages:
 
     print("validating packages...")
@@ -282,15 +302,28 @@ def main():
         pkg.created = batch_timestamp
         pkg.package_dir = pkg_dir
 
+
         if pkg.package_type == PackageType.core:
-            pass
+            pkg.out_rel_dir = PurePosixPath(".")
+            pkg.out_basename = pkg.package_name
+
         elif pkg.package_type == PackageType.build:
-            pass        
+            pkg.out_rel_dir = PurePosixPath(".")
+            pkg.out_basename = pkg.package_name
+
         elif pkg.package_type == PackageType.board_support:
-            pkg.inner_dependencies.add("core") # BSPs implicitly depend on the core "microzig" package
+            parsed_pkg_name = PurePosixPath(pkg.package_name)
+
+            pkg.out_rel_dir = "board-support" / parsed_pkg_name.parent
+            pkg.out_basename = parsed_pkg_name.name
+
+            pkg.inner_dependencies.add("microzig-core") # BSPs implicitly depend on the core "microzig" package
         else:
             assert False 
 
+        download_path = pkg.out_rel_dir / ALL_FILES_DIR / f"{pkg.out_basename}-{version}.tar.gz"
+        pkg.download_url = f"{DEPLOYMENT_BASE}/{download_path}"
+
         buildzig_path = pkg_dir / "build.zig"
         buildzon_path = pkg_dir / "build.zig.zon"
         
@@ -314,10 +347,6 @@ def main():
             print("Please remove that file and merge it into microzig-package.json!")
             validation_ok = False
         
-        # print("%d\t%s\n", "${pkg_prio}" "${reldir}" >> "${cache_dir}/packages.raw")
-
-    # cat "${cache_dir}/packages.raw" | sort | cut -f 2 > "${cache_dir}/packages.list"
-
     if not validation_ok:
         print("Not all packages are valid. Fix the packages and try again!" )
         exit(1)
@@ -332,8 +361,10 @@ def main():
 
     # bundle everything:
 
-    
+    index = PackageIndex(
-
+        last_update = batch_timestamp,
+        packages = [],
+    )
 
     print("creating packages...")
     for pkg in evaluation_ordered_packages:
@@ -347,24 +378,10 @@ def main():
         meta_path = pkg_dir / "microzig-package.json"
         pkg_zon_file = pkg_cache_dir / "build.zig.zon" 
 
-        out_rel_dir: PurePosixPath 
+        out_rel_dir: PurePosixPath = pkg.out_rel_dir
-        out_basename: str 
+        out_basename: str = pkg.out_basename
-        extra_json: dict = {}
-        
-        if pkg.package_type == PackageType.build:
-            out_rel_dir = PurePosixPath(".")
-            out_basename = pkg.package_name
-
-        elif pkg.package_type == PackageType.core:
-            out_rel_dir = PurePosixPath(".")
-            out_basename = pkg.package_name
-
-        elif pkg.package_type == PackageType.board_support:
-            parsed_pkg_name = PurePosixPath(pkg.package_name)
-
-            out_rel_dir = "board-support" / parsed_pkg_name.parent
-            out_basename = parsed_pkg_name.name
         
+        if pkg.package_type == PackageType.board_support:
             bsp_info = slurp(
                 "zig", "build-exe",
                     f"{REPO_ROOT}/tools/extract-bsp-info.zig" ,
@@ -379,14 +396,11 @@ def main():
 
             extra_json_str=slurp(pkg_cache_dir/"extract-bsp-info")
 
-            extra_json = json.loads(extra_json_str)
+            pkg.microzig = json.loads(extra_json_str)
 
-        else:
-            assert False 
 
         assert out_rel_dir is not None
         assert out_basename is not None
-        assert isinstance(extra_json, dict)
 
         # File names:
 
@@ -432,8 +446,9 @@ def main():
         execute("tar", "-cf", out_file_tar, "--hard-dereference", *package_files, cwd=pkg_dir)
 
         zon_data = slurp(
-            tools["create_pkg_descriptor"], version, out_rel_dir, 
+            tools["create_pkg_descriptor"],
-            input=PackageConfigurationSchema.dumps(pkg).encode(),
+            pkg.package_name, 
+            input=PackageConfigurationSchema.dumps(evaluation_ordered_packages, many=True ).encode(),
         )
 
         with pkg_zon_file.open("wb") as f:
@@ -465,6 +480,17 @@ def main():
         out_symlink_pkg.symlink_to(out_file_targz.relative_to(out_symlink_pkg.parent))
         out_symlink_meta.symlink_to(out_file_meta.relative_to(out_symlink_meta.parent))
 
+        index.packages.append(PackageDesc(
+            name = pkg.package_name,
+            type = pkg.package_type,
+            version = version,
+            metadata = pkg.download_url.removesuffix(".tar.gz") + ".json",
+            download = pkg.download_url,
+        ))
+
+    with (deploy_target / "index.json").open("w") as f:
+        f.write(PackageIndexSchema.dumps(index))
+
     # TODO: Verify that each package can be unpacked and built
 
 

tools/create-pkg-descriptor.zig

@@ -36,9 +36,27 @@ const MetaData = struct {
     archive: ?Archive = null,
     package: ?Package = null,
     created: ?Timestamp = null,
+
+    download_url: ?[]const u8 = null,
+
+    microzig: std.json.Value = .null,
 };
 
-// create-pkg-descriptor <version>
+fn findPackage(packages: []const MetaData, name: []const u8) ?*const MetaData {
+    return for (packages) |*pkg| {
+        if (std.mem.eql(u8, pkg.package_name, name))
+            return pkg;
+    } else null;
+}
+
+fn renderDep(writer: anytype, name: []const u8, url: []const u8, hash: []const u8) !void {
+    try writer.print("            .{} = .{{\n", .{fmtId(name)});
+    try writer.print("                .url = \"{}\",\n", .{fmtEscapes(url)});
+    try writer.print("                .hash = \"{}\",\n", .{fmtEscapes(hash)});
+    try writer.writeAll("        },\n");
+}
+
+// create-pkg-descriptor <package_name>
 pub fn main() !void {
     var gpa = std.heap.GeneralPurposeAllocator(.{}){};
     defer _ = gpa.deinit();
@@ -52,66 +70,54 @@ pub fn main() !void {
 
     const argv = try std.process.argsAlloc(arena);
 
-    if (argv.len != 3) {
+    if (argv.len != 2) {
         @panic("version and/or relpath missing!");
     }
 
-    // System configuration:
-    const deployment_base_url = "https://download.microzig.tech/packages"; // TODO: Make those configurable
-
     // build inputs:
-    const version_string = argv[1];
+    const pkg_name = argv[1];
-    const rel_pkg_path = argv[2];
-
-    const version = try std.SemanticVersion.parse(version_string);
 
     const json_input = try std.io.getStdIn().readToEndAlloc(arena, 1 << 20);
 
     errdefer std.log.err("failed to parse json from {s}", .{json_input});
 
-    const metadata = try std.json.parseFromSliceLeaky(MetaData, arena, json_input, .{});
+    const all_packages = try std.json.parseFromSliceLeaky([]MetaData, arena, json_input, .{});
+
+    const package = findPackage(all_packages, pkg_name).?;
+
+    const version = try std.SemanticVersion.parse(package.version);
 
     var buffered_stdout = std.io.bufferedWriter(std.io.getStdOut().writer());
 
     const stdout = buffered_stdout.writer();
     {
         try stdout.writeAll(".{\n");
-        try stdout.print("    .name = \"{}\",\n", .{fmtEscapes(metadata.package_name)});
+        try stdout.print("    .name = \"{}\",\n", .{fmtEscapes(package.package_name)});
         try stdout.print("    .version = \"{}\",\n", .{version});
         try stdout.writeAll("    .dependencies = .{\n");
-        if (metadata.external_dependencies != .null) {
+        if (package.external_dependencies != .null) {
-            const deps = &metadata.external_dependencies.object;
+            const deps = &package.external_dependencies.object;
             for (deps.keys(), deps.values()) |key, value| {
                 const dep: *const std.json.ObjectMap = &value.object;
-
+                try renderDep(
-                //
+                    stdout,
-
+                    key,
-                try stdout.print("            .{} = .{{\n", .{fmtId(key)});
+                    dep.get("url").?.string,
-
+                    dep.get("hash").?.string,
-                try stdout.print("                .url = \"{}\",\n", .{fmtEscapes(dep.get("url").?.string)});
+                );
-                try stdout.print("                .hash = \"{}\",\n", .{fmtEscapes(dep.get("hash").?.string)});
-                try stdout.writeAll("        },\n");
             }
         }
 
-        switch (metadata.package_type) {
+        // Add all other dependencies:
-            .core => {
+        for (package.inner_dependencies) |dep_name| {
-                // core packages are always "standalone" in the microzig environment and provide the root
+            const dep = findPackage(all_packages, dep_name).?;
-                // of the build
-            },
-
-            .build => {
-                //
-            },
-
-            .@"board-support" => {
-                // bsp packages implicitly depend on the "microzig" package:
 
-                try stdout.writeAll("            .microzig = .{\n");
+            try renderDep(
-                try stdout.print("                .url = \"{}/{}\",\n", .{ fmtEscapes(deployment_base_url), fmtEscapes(rel_pkg_path) });
+                stdout,
-                try stdout.print("                .hash = \"{}\",\n", .{fmtEscapes("???")});
+                "microzig",
-                try stdout.writeAll("        },\n");
+                dep.download_url.?,
-            },
+                dep.package.?.hash,
+            );
         }
 
         try stdout.writeAll("    },\n");