Merge branch 'fsi_dgl' of github.com:tzemicheal/Morpheus into fsi_dgl

.gitattributes

@@ -7,3 +7,7 @@ tests/mock_triton_server/payloads/** filter=lfs diff=lfs merge=lfs -text
 tests/tests_data/** filter=lfs diff=lfs merge=lfs -text
 examples/basic_usage/img/** filter=lfs diff=lfs merge=lfs -text
 docs/source/img/* filter=lfs diff=lfs merge=lfs -text
+git filter=lfs diff=lfs merge=lfs -text
+status filter=lfs diff=lfs merge=lfs -text
+examples/gnn_fraud_detection_pipeline/validation.csv filter=lfs diff=lfs merge=lfs -text
+examples/gnn_fraud_detection_pipeline/training.csv filter=lfs diff=lfs merge=lfs -text

.github/workflows/pull_request.yml

@@ -46,7 +46,7 @@ jobs:
     uses: ./.github/workflows/ci_pipe.yml
     with:
       run_check: ${{ startsWith(github.ref_name, 'pull-request/') }}
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-230801
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-230801
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-230828
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-230828
     secrets:
       NGC_API_KEY: ${{ secrets.NGC_API_KEY }}

ci/runner/Dockerfile

@@ -66,13 +66,6 @@ RUN apt update && \
 COPY ./docker/conda/environments/cuda${CUDA_SHORT_VER}_examples.yml /tmp/conda/cuda${CUDA_SHORT_VER}_examples.yml
 COPY ./ci/scripts/download_kafka.py /tmp/scripts/download_kafka.py
 
-# Install extra deps needed for gnn_fraud_detection_pipeline & ransomware_detection examples
-RUN CONDA_ALWAYS_YES=true /opt/conda/bin/mamba env update -n ${PROJ_NAME} -q --file /tmp/conda/cuda${CUDA_SHORT_VER}_examples.yml && \
-    conda clean -afy && \
-    source activate ${PROJ_NAME} && \
-    pip install --ignore-requires-python stellargraph==1.2.1 && \
-    rm -rf /tmp/conda
-
 # Install camouflage needed for unittests to mock a triton server
 RUN source activate ${PROJ_NAME} && \
     npm install -g [email protected] && \

ci/scripts/run_ci_local.sh

@@ -49,7 +49,7 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-230801}
+CONTAINER_VER=${CONTAINER_VER:-230828}
 CUDA_VER=${CUDA_VER:-11.8}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 

docker/conda/environments/cuda11.8_examples.yml

@@ -34,5 +34,3 @@ dependencies:
     - mlflow>=2.2.1,<3
     - papermill=2.3.4
     - s3fs>=2023.6
-    - pip
-    - wrapt=1.14.1 # ver 1.15 breaks the keras model used by the gnn_fraud_detection_pipeline

examples/gnn_fraud_detection_pipeline/README.md

@@ -120,9 +120,8 @@ morpheus --log_level INFO \
 	deserialize \
 	fraud-graph-construction --training_file examples/gnn_fraud_detection_pipeline/training.csv \
 	monitor --description "Graph construction rate" \
-	gnn-fraud-sage --model_hinsage_file examples/gnn_fraud_detection_pipeline/model/hinsage-model.pt \
+	gnn-fraud-sage --model_dir  examples/gnn_fraud_detection_pipeline/model/ \
 	monitor --description "Inference rate" \
-	gnn-fraud-classification --model_xgb_file examples/gnn_fraud_detection_pipeline/model/xgb-model.pt \
 	monitor --description "Add classification rate" \
 	serialize \
 	to-file --filename "output.csv" --overwrite

examples/gnn_fraud_detection_pipeline/run.py

@@ -27,6 +27,7 @@
 from morpheus.stages.postprocess.serialize_stage import SerializeStage
 from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
 from morpheus.utils.logger import configure_logging
+# pylint: disable=no-name-in-module
 from stages.classification_stage import ClassificationStage
 from stages.graph_construction_stage import FraudGraphConstructionStage
 from stages.graph_sage_stage import GraphSAGEStage
@@ -60,27 +61,28 @@
 )
 @click.option(
     "--input_file",
-    type=click.Path(exists=True, readable=True),
+    type=click.Path(exists=True, readable=True, dir_okay=False),
     default="validation.csv",
     required=True,
     help="Input data filepath.",
 )
 @click.option(
     "--training_file",
-    type=click.Path(exists=True, readable=True),
+    type=click.Path(exists=True, readable=True, dir_okay=False),
     default="training.csv",
     required=True,
     help="Training data filepath.",
 )
 @click.option(
     "--model_dir",
-    type=click.Path(exists=True, readable=True),
+    type=click.Path(exists=True, readable=True, file_okay=False, dir_okay=True),
     default="model",
     required=True,
-    help="Trained hinsage model directory path.",
+    help="Path to trained Hinsage & XGB models.",
 )
 @click.option(
     "--output_file",
+    type=click.Path(dir_okay=False),
     default="output.csv",
     help="The path to the file where the inference output will be saved.",
 )

examples/gnn_fraud_detection_pipeline/stages/classification_stage.py

@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import typing
-
 import mrc
 from mrc.core import operators as ops
 
@@ -55,13 +53,13 @@ def __init__(self, c: Config, model_xgb_file: str):
     def name(self) -> str:
         return "gnn-fraud-classification"
 
-    def accepted_types(self) -> typing.Tuple:
+    def accepted_types(self) -> (GraphSAGEMultiMessage, ):
         return (GraphSAGEMultiMessage, )
 
-    def supports_cpp_node(self):
+    def supports_cpp_node(self) -> bool:
         return False
 
-    def _process_message(self, message: GraphSAGEMultiMessage):
+    def _process_message(self, message: GraphSAGEMultiMessage) -> GraphSAGEMultiMessage:
         ind_emb_columns = message.get_meta(message.inductive_embedding_column_names)
         message.set_meta("node_id", message.node_identifiers)
 

examples/gnn_fraud_detection_pipeline/stages/graph_construction_stage.py

@@ -15,7 +15,6 @@
 
 import dataclasses
 import pathlib
-import typing
 
 import dgl
 import mrc
@@ -76,13 +75,13 @@ def __init__(self, config: Config, training_file: pathlib.Path):
     def name(self) -> str:
         return "fraud-graph-construction"
 
-    def accepted_types(self) -> typing.Tuple:
+    def accepted_types(self) -> (MultiMessage, ):
         return (MultiMessage, )
 
-    def supports_cpp_node(self):
+    def supports_cpp_node(self) -> bool:
         return False
 
-    def _process_message(self, message: MultiMessage):
+    def _process_message(self, message: MultiMessage) -> FraudGraphMultiMessage:
 
         _, _, _, test_index, _, graph_data = prepare_data(self._training_data, message.get_meta(self._column_names))
 

examples/gnn_fraud_detection_pipeline/stages/graph_sage_stage.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import dataclasses
-import typing
 
 import mrc
 from mrc.core import operators as ops
@@ -36,16 +35,16 @@
 
 @dataclasses.dataclass
 class GraphSAGEMultiMessage(MultiMessage):
-    node_identifiers: typing.List[int]
-    inductive_embedding_column_names: typing.List[str]
+    node_identifiers: list[int]
+    inductive_embedding_column_names: list[str]
 
     def __init__(self,
                  *,
                  meta: MessageMeta,
                  mess_offset: int = 0,
                  mess_count: int = -1,
-                 node_identifiers: typing.List[int],
-                 inductive_embedding_column_names: typing.List[str]):
+                 node_identifiers: list[int],
+                 inductive_embedding_column_names: list[str]):
         super().__init__(meta=meta, mess_offset=mess_offset, mess_count=mess_count)
 
         self.node_identifiers = node_identifiers
@@ -72,13 +71,13 @@ def __init__(self,
     def name(self) -> str:
         return "gnn-fraud-sage"
 
-    def accepted_types(self) -> typing.Tuple:
+    def accepted_types(self) -> (FraudGraphMultiMessage, ):
         return (FraudGraphMultiMessage, )
 
-    def supports_cpp_node(self):
+    def supports_cpp_node(self) -> bool:
         return False
 
-    def _process_message(self, message: FraudGraphMultiMessage):
+    def _process_message(self, message: FraudGraphMultiMessage) -> GraphSAGEMultiMessage:
 
         node_identifiers = list(message.get_meta(self._record_id).to_pandas())