Merge pull request #56 from compose-viz/dev

Fix #51 #54

.github/workflows/ci.yml

@@ -26,9 +26,9 @@ jobs:
           python-version: '3.10.4'
 
       - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v7
+        uses: abatilo/actions-poetry@v3
         with:
-          poetry-version: 1.2.1
+          poetry-version: 1.8.2
 
       - name: Install Dependencies
         run: |

.github/workflows/release-tagged-version.yml

@@ -32,9 +32,9 @@ jobs:
           python-version: '3.10.4'
 
       - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v7
+        uses: abatilo/actions-poetry@v3
         with:
-          poetry-version: 1.2.1
+          poetry-version: 1.8.2
       - run: |
           poetry install --no-root
           poetry build

.github/workflows/update-submodules.yml

@@ -0,0 +1,79 @@
+name: Update Submodules
+
+on:
+  push:
+    branches: [ dev ]
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  check_submodules:
+    name: Check Submodules
+    runs-on: ubuntu-latest
+    outputs:
+      has_changes: ${{ steps.check.outputs.has_changes }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Create new branch and push changes
+        run: |
+          git submodule update --remote
+
+      - name: Check for changes
+        id: check
+        run: |
+          git diff --quiet || echo "::set-output name=has_changes::true"
+
+  update_submodules:
+    name: Update Submodules
+    runs-on: ubuntu-latest
+    needs: [check_submodules]
+    if: needs.check_submodules.outputs.has_changes == 'true'
+    steps:
+
+      - name: Setup Python 3.10.4
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.10.4'
+
+      - name: Setup Poetry
+        uses: abatilo/actions-poetry@v3
+        with:
+          poetry-version: 1.8.2
+
+      - name: Install Dependencies
+        run: |
+          poetry install --no-root
+
+      - name: Update Submodule
+        run: |
+          datamodel-codegen --input ./compose-spec/schema/compose-spec.json --output-model-type pydantic_v2.BaseModel --field-constraints --output ./compose_viz/spec/compose_spec.py
+          poetry run python ./update-submodules.py
+
+      - name: Execute pre-commit
+        continue-on-error: true
+        run: |
+          poetry run python -m pre_commit run --all-files
+
+      - name: Push changes
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git checkout -b $GITHUB_RUN_ID
+          git commit -am "chore: update submodules"
+          git push --set-upstream origin $GITHUB_RUN_ID
+
+      - name: File PR
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.pulls.create({
+              owner: '${{ github.repository_owner }}',
+              repo: 'compose-viz',
+              head: process.env.GITHUB_RUN_ID,
+              base: 'main',
+              title: `chore: update submodules (${process.env.GITHUB_RUN_ID})`,
+              body: `Please add the version tag to trigger the release.`,
+            });

README.md

@@ -122,6 +122,7 @@ Check out the result [here](https://github.com/compose-viz/compose-viz/blob/main
 | `-o, --output-filename FILENAME`  | Output filename for the generated visualization file. [default: compose-viz]                                                                                                        |
 | `-m, --format FORMAT`             | Output format for the generated visualization file. See [supported formats](https://github.com/compose-viz/compose-viz/blob/main/compose_viz/models/viz_formats.py). [default: png] |
 | `-r, --root-service SERVICE_NAME` | Root of the service tree (convenient for large compose yamls)                                                                                                                       |
+| `-l, --legend`                    | Include a legend in the visualization.                                                                                                                                              |
 | `-v, --version`                   | Show the version of compose-viz.                                                                                                                                                    |
 | `--help`                          | Show help and exit.                                                                                                                                                                 |
 

compose_viz/__init__.py

@@ -1,2 +1,2 @@
-__app_name__ = "compose_viz"
+__app_name__ = "compose_viz"
-__version__ = "0.3.1"
+__version__ = "0.3.2"

compose_viz/cli.py

@@ -42,6 +42,12 @@ def compose_viz(
         "-r",
         help="Root of the service tree (convenient for large compose yamls)",
     ),
+    include_legend: bool = typer.Option(
+        False,
+        "--legend",
+        "-l",
+        help="Include a legend in the visualization.",
+    ),
     _: Optional[bool] = typer.Option(
         None,
         "--version",
@@ -57,7 +63,7 @@ def compose_viz(
     if compose:
         typer.echo(f"Successfully parsed {input_path}")
 
-    Graph(compose, output_filename).render(format)
+    Graph(compose, output_filename, include_legend).render(format)
 
     raise typer.Exit()
 

compose_viz/graph.py

@@ -3,6 +3,7 @@ from typing import Optional
 import graphviz
 
 from compose_viz.models.compose import Compose
+from compose_viz.models.port import AppProtocol, Protocol
 
 
 def apply_vertex_style(type: str) -> dict:
@@ -69,12 +70,58 @@ def apply_edge_style(type: str) -> dict:
 
 
 class Graph:
-    def __init__(self, compose: Compose, filename: str) -> None:
+    def __init__(self, compose: Compose, filename: str, include_legend: bool) -> None:
         self.dot = graphviz.Digraph()
         self.dot.attr("graph", background="#ffffff", pad="0.5", ratio="fill")
         self.compose = compose
         self.filename = filename
 
+        if include_legend:
+            self.dot.attr(rankdir="LR")
+
+            with self.dot.subgraph(name="cluster_edge_") as edge:
+                edge.attr(label="Edge")
+                edge.node("line_0_l", style="invis")
+                edge.node("line_0_r", style="invis")
+                edge.edge("line_0_l", "line_0_r", label="exposes", **apply_edge_style("exposes"))
+
+                edge.node("line_1_l", style="invis")
+                edge.node("line_1_r", style="invis")
+                edge.edge("line_1_l", "line_1_r", label="links", **apply_edge_style("links"))
+
+                edge.node("line_2_l", style="invis")
+                edge.node("line_2_r", style="invis")
+                edge.edge("line_2_l", "line_2_r", label="volumes_rw", **apply_edge_style("volumes_rw"))
+
+                edge.node("line_3_l", style="invis")
+                edge.node("line_3_r", style="invis")
+                edge.edge("line_3_l", "line_3_r", label="volumes_ro", **apply_edge_style("volumes_ro"))
+
+                edge.node("line_4_l", style="invis")
+                edge.node("line_4_r", style="invis")
+                edge.edge("line_4_l", "line_4_r", label="depends_on", **apply_edge_style("depends_on"))
+
+                edge.node("line_5_l", style="invis")
+                edge.node("line_5_r", style="invis")
+                edge.edge("line_5_l", "line_5_r", label="extends", **apply_edge_style("extends"))
+
+            with self.dot.subgraph(name="cluster_node_") as node:
+                node.attr(label="Node")
+                node.node("service", shape="component", label="Service\n(image)")
+                node.node("volume", shape="cylinder", label="Volume")
+                node.node("network", shape="pentagon", label="Network")
+                node.node("port", shape="circle", label="Port")
+                node.node("env_file", shape="tab", label="Env File")
+                node.node("profile", shape="invhouse", label="Profile")
+                node.node("cgroup", shape="diamond", label="CGroupe")
+                node.node("device", shape="box3d", label="Device")
+
+                node.body.append("{ rank=source;service network env_file cgroup }")
+
+            self.dot.node("inv", style="invis")
+            self.dot.edge("inv", "network", style="invis")
+            self.dot.edge("port", "line_2_l", style="invis")
+
     def validate_name(self, name: str) -> str:
         # graphviz does not allow ':' in node name
         transTable = name.maketrans({":": ""})
@@ -117,7 +164,14 @@ class Graph:
                 self.add_edge(expose, service.name, "exposes")
             for port in service.ports:
                 self.add_vertex(port.host_port, "port", lable=port.host_port)
-                self.add_edge(port.host_port, service.name, "links", lable=port.container_port)
+                self.add_edge(
+                    port.host_port,
+                    service.name,
+                    "links",
+                    lable=port.container_port
+                    + (("/" + port.protocol) if port.protocol != Protocol.any.value else "")
+                    + (("\n(" + port.app_protocol + ")") if port.app_protocol != AppProtocol.na.value else ""),
+                )
             for env_file in service.env_file:
                 self.add_vertex(env_file, "env_file")
                 self.add_edge(env_file, service.name, "env_file")

compose_viz/models/port.py

@@ -7,11 +7,27 @@ class Protocol(str, Enum):
     any = "any"
 
 
+class AppProtocol(str, Enum):
+    rest = "REST"
+    mqtt = "MQTT"
+    wbsock = "WebSocket"
+    http = "http"
+    https = "https"
+    na = "NA"
+
+
 class Port:
-    def __init__(self, host_port: str, container_port: str, protocol: Protocol = Protocol.any):
+    def __init__(
+        self,
+        host_port: str,
+        container_port: str,
+        protocol: Protocol = Protocol.any,
+        app_protocol: AppProtocol = AppProtocol.na,
+    ):
         self._host_port = host_port
         self._container_port = container_port
         self._protocol = protocol
+        self._app_protocol = app_protocol
 
     @property
     def host_port(self):
@@ -24,3 +40,7 @@ class Port:
     @property
     def protocol(self):
         return self._protocol
+
+    @property
+    def app_protocol(self):
+        return self._app_protocol

compose_viz/parser.py

@@ -1,14 +1,13 @@
 import re
 from typing import Any, Dict, List, Optional, Union
 
-from pydantic import ValidationError
 from pydantic_yaml import parse_yaml_raw_as
 
 import compose_viz.spec.compose_spec as spec
 from compose_viz.models.compose import Compose, Service
 from compose_viz.models.device import Device
 from compose_viz.models.extends import Extends
-from compose_viz.models.port import Port, Protocol
+from compose_viz.models.port import AppProtocol, Port, Protocol
 from compose_viz.models.volume import Volume, VolumeType
 
 
@@ -44,7 +43,7 @@ class Parser:
             with open(file_path, "r") as file:
                 file_content = file.read()
             compose_data = parse_yaml_raw_as(spec.ComposeSpecification, file_content)
-        except ValidationError as e:
+        except Exception as e:
             raise RuntimeError(f"Error parsing file '{file_path}': {e}")
 
         services: List[Service] = []
@@ -102,12 +101,13 @@ class Parser:
                     host_port: Optional[str] = None
                     container_port: Optional[str] = None
                     protocol: Optional[str] = None
+                    app_protocol: Optional[str] = None
 
                     if type(port_data) is float:
                         container_port = str(int(port_data))
                         host_port = f"0.0.0.0:{container_port}"
                     elif type(port_data) is str:
-                        regex = r"((?P<host_ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:)|:)?((?P<host_port>\d+(\-\d+)?):)?((?P<container_port>\d+(\-\d+)?))?(/(?P<protocol>\w+))?"  # noqa: E501
+                        regex = r"((?P<host_ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:|(\$\{([^}]+)\}):)|:|)?((?P<host_port>\d+(\-\d+)?):)?((?P<container_port>\d+(\-\d+)?))?(/(?P<protocol>\w+))?"  # noqa: E501
                         match = re.match(regex, port_data)
 
                         if match:
@@ -128,17 +128,15 @@ class Parser:
                     elif type(port_data) is spec.Ports:
                         assert port_data.target is not None, "Invalid port format, aborting."
 
-                        # ruamel.yaml does not parse port as int
+                        if type(port_data.published) is str or type(port_data.published) is int:
-                        assert type(port_data.published) is not int
+                            host_port = str(port_data.published)
-
-                        if type(port_data.published) is str:
-                            host_port = port_data.published
 
                         if type(port_data.target) is int:
                             container_port = str(port_data.target)
 
                         host_ip = port_data.host_ip
                         protocol = port_data.protocol
+                        app_protocol = port_data.app_protocol
 
                         if container_port is not None and host_port is None:
                             host_port = container_port
@@ -154,11 +152,15 @@ class Parser:
                     if protocol is None:
                         protocol = "any"
 
+                    if app_protocol is None:
+                        app_protocol = "na"
+
                     service_ports.append(
                         Port(
                             host_port=host_port,
                             container_port=container_port,
                             protocol=Protocol[protocol],
+                            app_protocol=AppProtocol[app_protocol],
                         )
                     )
 
@@ -215,11 +217,16 @@ class Parser:
 
             env_file: List[str] = []
             if service_data.env_file is not None:
-                if type(service_data.env_file) is spec.StringOrList:
+                if type(service_data.env_file.root) is str:
-                    if type(service_data.env_file.root) is spec.ListOfStrings:
+                    env_file = [service_data.env_file.root]
-                        env_file = service_data.env_file.root.root
+                elif type(service_data.env_file.root) is list:
-                    elif type(service_data.env_file.root) is str:
+                    for env_file_data in service_data.env_file.root:
-                        env_file.append(service_data.env_file.root)
+                        if type(env_file_data) is str:
+                            env_file.append(env_file_data)
+                        elif type(env_file_data) is spec.EnvFilePath:
+                            env_file.append(env_file_data.path)
+                else:
+                    print(f"Invalid env_file data: {service_data.env_file.root}")
 
             expose: List[str] = []
             if service_data.expose is not None:

examples/non-normative/docker-compose.yml

@@ -42,7 +42,13 @@ services:
     extends:
       service: frontend
     ports:
-      - "8000:5010"
+      - name: web-secured
+        target: 443
+        host_ip: 127.0.0.1
+        published: "8083-9000"
+        protocol: tcp
+        app_protocol: wbsock
+        mode : host
     links:
       - "db:database"
     cgroup_parent: awesome-parent

pyproject.toml

@@ -1,36 +1,36 @@
-[tool.poetry]
+[tool.poetry]
-name = "compose-viz"
+name = "compose-viz"
-version = "0.3.1"
+version = "0.3.2"
-description = "A compose file visualization tool that supports compose-spec and allows you to gernerate graph in several formats."
+description = "A compose file visualization tool that supports compose-spec and allows you to gernerate graph in several formats."
-authors = ["Xyphuz Wu <xyphuzwu@gmail.com>"]
+authors = ["Xyphuz Wu <xyphuzwu@gmail.com>"]
-readme = "README.md"
+readme = "README.md"
-license = "MIT"
+license = "MIT"
-homepage = "https://github.com/compose-viz/compose-viz"
+homepage = "https://github.com/compose-viz/compose-viz"
-repository = "https://github.com/compose-viz/compose-viz"
+repository = "https://github.com/compose-viz/compose-viz"
-include = [
+include = [
-    "LICENSE",
+    "LICENSE",
-]
+]
-
+
-[tool.poetry.dependencies]
+[tool.poetry.dependencies]
-python = "^3.9"
+python = "^3.9"
-typer = "^0.4.1"
+typer = "^0.4.1"
-graphviz = "^0.20"
+graphviz = "^0.20"
-pydantic-yaml = "^1.3.0"
+pydantic-yaml = "^1.3.0"
-
+
-[tool.poetry.group.dev.dependencies]
+[tool.poetry.group.dev.dependencies]
-pytest = "^8.1.2"
+pytest = "^8.1.2"
-pre-commit = "^3.7.0"
+pre-commit = "^3.7.0"
-coverage = "^7.5.0"
+coverage = "^7.5.0"
-pytest-cov = "^5.0.0"
+pytest-cov = "^5.0.0"
-datamodel-code-generator = "^0.25.6"
+datamodel-code-generator = "^0.25.6"
-
+
-[build-system]
+[build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"
-
+
-[tool.poetry.scripts]
+[tool.poetry.scripts]
-cpv = "compose_viz.cli:start_cli"
+cpv = "compose_viz.cli:start_cli"
-
+
-[tool.coverage.run]
+[tool.coverage.run]
-source = ["compose_viz"]
+source = ["compose_viz"]
-omit = ["compose_viz/spec/*"]
+omit = ["compose_viz/spec/*"]

tests/test_parse_file.py

@@ -176,6 +176,10 @@ from compose_viz.parser import Parser
                                 host_port="0.0.0.0:7777",
                                 container_port="7777",
                             ),
+                            Port(
+                                host_port="${BIND_IP:-127.0.0.1}:8080",
+                                container_port="8080",
+                            ),
                             Port(
                                 host_port="127.0.0.1:8080",
                                 container_port="80",

tests/ymls/networks/docker-compose.yml

@@ -26,5 +26,3 @@ networks:
   front-tier:
   back-tier:
   admin:
-  traefik-public:
-    external: true

tests/ymls/ports/docker-compose.yml

@@ -12,6 +12,7 @@ services:
       - "127.0.0.1:5000-5010:5000-5010"
       - "6060:6060/udp"
       - ":7777"
+      - "${BIND_IP:-127.0.0.1}:8080:8080"
       - target: 80
         host_ip: 127.0.0.1
         published: 8080

update-submodules.py

@@ -0,0 +1,54 @@
+import re
+
+
+def revise_naming_convention():
+    name_mapping = {
+        "EnvFile1": "EnvFilePath",
+        "Volume1": "AdditionalVolumeOption",
+        "External": "ExternalVolumeNetwork",
+        "External2": "ExternalConfig",
+    }
+
+    spec_content: str
+    with open("./compose_viz/spec/compose_spec.py", "r+") as spec_file:
+        spec_content: str = spec_file.read()
+
+        for origin_name, new_name in name_mapping.items():
+            spec_content = re.sub(rf"\b{origin_name}\b", new_name, spec_content)
+
+        spec_file.seek(0)
+        spec_file.write(spec_content)
+
+    print("Revised naming convention successfully!")
+
+
+def update_version_number():
+    new_version_number: str
+    with open("./compose_viz/__init__.py", "r+") as init_file:
+        init_content: str = init_file.read()
+
+        version_number = init_content.split(" ")[-1].replace('"', "").strip()
+        major, minor, patch = version_number.split(".")
+        new_version_number = f"{major}.{minor}.{int(patch) + 1}"
+
+        init_file.seek(0)
+        init_file.write(
+            f"""__app_name__ = "compose_viz"
+__version__ = "{new_version_number}"
+"""
+        )
+
+    with open("./pyproject.toml", "r+") as pyproject_file:
+        pyproject_content: str = pyproject_file.read()
+
+        pyproject_content = pyproject_content.replace(version_number, new_version_number)
+
+        pyproject_file.seek(0)
+        pyproject_file.write(pyproject_content)
+
+    print(f"Version number updated to {new_version_number} successfully!")
+
+
+if __name__ == "__main__":
+    revise_naming_convention()
+    update_version_number()