Added field to multi-file signature; the steps are documented in adding-to-typegraph.org

data/treeio/multi-sarif-01.json

@@ -1,26 +1 @@
-[
+[{"creation_date": "2021-12-09", "primary_language": "javascript", "project_name": "treeio/treeio", "query_commit_id": "fa9571646c", "sarif_content": {}, "sarif_file_name": "2021-12-09/results.sarif", "scan_id": 123456, "scan_start_date": "2021-12-09", "scan_stop_date": "2021-12-10", "tool_name": "codeql", "tool_version": "v1.27"}, {"creation_date": "2022-02-25", "primary_language": "javascript", "project_name": "treeio/treeio", "query_commit_id": "fa9571646c", "sarif_content": {}, "sarif_file_name": "2022-02-25/results.sarif", "scan_id": 123457, "scan_start_date": "2022-02-25", "scan_stop_date": "2022-02-26", "tool_name": "codeql", "tool_version": "v1.29"}]
-  {
-    "creation_date": "2021-12-09",
-    "primary_language": "javascript",
-    "project_name": "treeio/treeio",
-    "query_commit_id": "fa9571646c",
-    "sarif_content": {},
-    "sarif_file_name": "2021-12-09/results.sarif",
-    "scan_start_date": "2021-12-09",
-    "scan_stop_date": "2021-12-10",
-    "tool_name": "codeql",
-    "tool_version": "v1.27"
-  },
-  {
-    "creation_date": "2022-02-25",
-    "primary_language": "javascript",
-    "project_name": "treeio/treeio",
-    "query_commit_id": "fa9571646c",
-    "sarif_content": {},
-    "sarif_file_name": "2022-02-25/results.sarif",
-    "scan_start_date": "2022-02-25",
-    "scan_stop_date": "2022-02-26",
-    "tool_name": "codeql",
-    "tool_version": "v1.29"
-  }
-]

data/treeio/multi-sarif-01.yaml

@@ -4,6 +4,7 @@
   query_commit_id: fa9571646c   # Commit id for custom (non-library) queries
   sarif_content: {}             # The sarif content will be attached here
   sarif_file_name: 2021-12-09/results.sarif # Path to sarif file
+  scan_id: 123456                           # sequential id, likely from postgres / mysql
   scan_start_date: '2021-12-09'             # Beginning date/time of scan
   scan_stop_date:  '2021-12-10'             # End date/time of scan
   tool_name: codeql
@@ -15,6 +16,7 @@
   query_commit_id: fa9571646c  
   sarif_content: {}            
   sarif_file_name: 2022-02-25/results.sarif 
+  scan_id: 123457                           # sequential id, likely from postgres / mysql
   scan_start_date: '2022-02-25'  
   scan_stop_date:  '2022-02-26'  
   tool_name: codeql

notes/adding-to-typegraph.org

@@ -0,0 +1,57 @@
+* Adding to / updating the typegraph
+
+  1. Open the current graph in [[./typegraph-multi.pdf]] as reference
+  2. Identify the element to update, e.g. Struct3739
+  3. Find its uses via
+     #+BEGIN_SRC sh
+       ag 3739 bin sarif_cli
+     #+END_SRC
+     or so.
+  4. Update or follow the steps in the found files; in this case
+     #+BEGIN_SRC text
+       sarif_cli/table_joins.py
+       228:        .merge(sf(3739), how="left", left_on='id_or_value_at_index', right_on='struct_id', validate="1:m")
+
+       sarif_cli/signature_multi.py
+       135:    (   'Struct3739',
+       147:    ('Array6785', ('array', (0, 'Struct3739')))]
+     #+END_SRC
+     1. First update the signature.  The file [[./sarif_cli/signature_multi.py]] has
+        instructions for updating (or creating) a typegraph.
+
+        The update from commit 0f070a6ae to 0f070a6ae+1 introduces the changes
+        #+BEGIN_SRC text
+          -    (   'Struct3739',
+          +    (   'Struct3452',
+
+          -    ('Array6785', ('array', (0, 'Struct3739')))]
+          +    ('Array7481', ('array', (0, 'Struct3452')))]
+        #+END_SRC
+
+     2. Update the existing uses.  Ids are unique, so a simple search is enough:
+        #+BEGIN_SRC sh
+          cd ~/local/sarif-cli
+          ag '(3739|6785)' bin sarif_cli
+
+          sarif_cli/table_joins.py
+          226:        af(6785)
+          228:        .merge(sf(3739), how="left", left_on='id_or_value_at_index', right_on='struct_id', validate="1:m")
+          236:               suffixes=("_6785", "_0177"), validate="1:m")
+
+          sarif_cli/signature_multi.py
+          26:start_node_2022_03_08 = 'Array6785'
+        #+END_SRC
+
+  5. Update the typegraph's pdf.  For this case,
+     #+BEGIN_SRC sh
+       cd ../data/treeio
+       sarif-extract-multi -c multi-sarif-01.json none | \
+           sarif-to-dot -u -t -f -n -d  - | dot -Tpdf > typegraph-multi.pdf
+
+       mv typegraph-multi.pdf ../../notes/
+     #+END_SRC
+
+  6. Run the tests.  Currently, =*test*= files in [[../scripts/]].
+
+  
+      

sarif_cli/signature_multi.py

@@ -23,7 +23,7 @@ To get a map of this type graph, use
 # 
 # The starting node is the leftmost node in ../notes/typegraph-multi.pdf
 # 
-start_node_2022_03_08 = 'Array6785'
+start_node_2022_03_08 = 'Array7481'
 
 struct_graph_2022_03_08 = (
 [   ('String', 'string'),
@@ -131,8 +131,8 @@ struct_graph_2022_03_08 = (
         (   'struct',
             ('$schema', 'String'),
             ('runs', 'Array0177'),
-            ('version', 'String'))), # Up to here identical to struct_graph_2022_02_01
+            ('version', 'String'))),    # Up to here identical to struct_graph_2022_02_01
-    (   'Struct3739',
+    (   'Struct3452',
         (   'struct',
             ('creation_date', 'String'),
             ('primary_language', 'String'),
@@ -140,9 +140,10 @@ struct_graph_2022_03_08 = (
             ('query_commit_id', 'String'),
             ('sarif_content', 'Struct6787'),
             ('sarif_file_name', 'String'),
+            ('scan_id', 'Int'),
             ('scan_start_date', 'String'),
             ('scan_stop_date', 'String'),
             ('tool_name', 'String'),
             ('tool_version', 'String'))),
-    ('Array6785', ('array', (0, 'Struct3739')))]
+    ('Array7481', ('array', (0, 'Struct3452')))]
 )

sarif_cli/table_joins.py

@@ -223,9 +223,9 @@ def joins_for_project(tgraph):
     af = lambda num: tgraph.dataframes['Array' + str(num)]
     # 
     project_df = (
-        af(6785)
+        af(7481)
         #
-        .merge(sf(3739), how="left", left_on='id_or_value_at_index', right_on='struct_id', validate="1:m")
+        .merge(sf(3452), how="left", left_on='id_or_value_at_index', right_on='struct_id', validate="1:m")
         .drop(columns=['id_or_value_at_index', 'struct_id', 'array_id', 'type_at_index'])
         #
         .merge(sf(6787), how="left", left_on='sarif_content', right_on='struct_id', validate="1:m")
@@ -233,7 +233,7 @@ def joins_for_project(tgraph):
         .rename(columns={"version": "version_6787"})
         #
         .merge(af('0177'), how="left", left_on='runs', right_on='array_id',
-               suffixes=("_6785", "_0177"), validate="1:m")
+               suffixes=("_7481", "_0177"), validate="1:m")
         .drop(columns=['runs', 'array_id', 'type_at_index'])
         #
         .merge(sf(3388), how="left", left_on='id_or_value_at_index', right_on='struct_id', validate="1:m")

scripts/file-level-tests.sh

@@ -15,7 +15,7 @@ sarif-results-summary  -h
 test_files="
 ../data/wxWidgets_wxWidgets__2021-11-21_16_06_30__export.sarif
 ../data/torvalds_linux__2021-10-21_10_07_00__export.sarif
-../data/treeio/results.sarif
+../data/treeio/2021-12-09/results.sarif
 "
 for file in $test_files ; do
     sarif-results-summary $file > /dev/null 

scripts/table-tests.sh

@@ -3,9 +3,5 @@
 # Sanity tests for the table-producing scripts.  Should succeed and produce
 # nothing on stdout/stderr
 # 
-
+( cd ../data/treeio/2021-12-09 && sarif-extract-tables results.sarif test-tables )
-cd ~/local/sarif-cli/data/treeio/2021-12-09
+( cd ../data/treeio && sarif-extract-multi multi-sarif-01.json test-multi-table )
-sarif-extract-tables results.sarif test-tables
-
-cd ~/local/sarif-cli/data/treeio
-sarif-extract-multi multi-sarif-01.json test-multi-table