added mongo for json storage

2025-02-22 07:52:59 +08:00 · 2024-08-01 00:36:47 +02:00 · 2024-08-01 00:36:47 +02:00 · 66656f23f3
commit 66656f23f3
parent 305ead15a6
7 changed files with 225 additions and 83 deletions
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@ -5,41 +5,13 @@
  </component>
  <component name="ChangeListManager">
    <list default="true" id="9f590b69-f929-45a1-8512-12ed6efbf028" name="Changes" comment="Added Invetory starter">
+      <change afterPath="$PROJECT_DIR$/felicity/database/mongo.py" afterDir="false" />
      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
      <change beforePath="$PROJECT_DIR$/.vscode/PythonImportHelper-v2-Completion.json" beforeDir="false" afterPath="$PROJECT_DIR$/.vscode/PythonImportHelper-v2-Completion.json" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/Dockerfile.dev" beforeDir="false" afterPath="$PROJECT_DIR$/Dockerfile.dev" afterDir="false" />
      <change beforePath="$PROJECT_DIR$/docker-compose.dev.yml" beforeDir="false" afterPath="$PROJECT_DIR$/docker-compose.dev.yml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/analysis/mutations/analysis_request.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/analysis/mutations/analysis_request.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/analysis/mutations/quality_control.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/analysis/mutations/quality_control.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/auth/__init__.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/auth/__init__.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/billing/mutations.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/billing/mutations.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/billing/types.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/billing/types.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/impress/query.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/impress/query.py" afterDir="false" />
      <change beforePath="$PROJECT_DIR$/felicity/api/gql/impress/types.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/impress/types.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/inventory/types.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/inventory/types.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/notification/subscription.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/notification/subscription.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/notification/types.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/notification/types.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/patient/mutations.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/patient/mutations.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/patient/types.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/patient/types.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/gql/storage/types.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/gql/storage/types.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/api/rest/api_v1/fhir/r4.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/api/rest/api_v1/fhir/r4.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/apps/abstract/entity.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/apps/abstract/entity.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/apps/abstract/repository.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/apps/abstract/repository.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/apps/abstract/service.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/apps/abstract/service.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/apps/impress/entities.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/apps/impress/entities.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/apps/impress/sample/schemas.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/apps/impress/sample/schemas.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/apps/impress/sample/tasks.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/apps/impress/sample/tasks.py" afterDir="false" />
      <change beforePath="$PROJECT_DIR$/felicity/apps/impress/sample/utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/apps/impress/sample/utils.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/apps/setup/caches.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/apps/setup/caches.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/felicity/tests/integration/conftest.py" beforeDir="false" afterPath="$PROJECT_DIR$/felicity/tests/integration/conftest.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/mypy.ini" beforeDir="false" afterPath="$PROJECT_DIR$/mypy.ini" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/requirements-dev.txt" beforeDir="false" afterPath="$PROJECT_DIR$/requirements-dev.txt" afterDir="false" />
      <change beforePath="$PROJECT_DIR$/requirements.txt" beforeDir="false" afterPath="$PROJECT_DIR$/requirements.txt" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/runtime.txt" beforeDir="false" afterPath="$PROJECT_DIR$/runtime.txt" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/webapp/components/notification/FelNotification.vue" beforeDir="false" afterPath="$PROJECT_DIR$/webapp/components/notification/FelNotification.vue" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/webapp/composables/samples.ts" beforeDir="false" afterPath="$PROJECT_DIR$/webapp/composables/samples.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/webapp/graphql/operations/analyses.queries.ts" beforeDir="false" afterPath="$PROJECT_DIR$/webapp/graphql/operations/analyses.queries.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/webapp/views/sample/_id/Impress.vue" beforeDir="false" afterPath="$PROJECT_DIR$/webapp/views/sample/_id/Impress.vue" afterDir="false" />
    </list>
    <option name="SHOW_DIALOG" value="false" />
    <option name="HIGHLIGHT_CONFLICTS" value="true" />
--- a/.vscode/PythonImportHelper-v2-Completion.json
+++ b/.vscode/PythonImportHelper-v2-Completion.json
@ -1537,6 +1537,14 @@
        "detail": "typing",
        "documentation": {}
    },
+    {
+        "label": "Optional",
+        "importPath": "typing",
+        "description": "typing",
+        "isExtraImport": true,
+        "detail": "typing",
+        "documentation": {}
+    },
    {
        "label": "List",
        "importPath": "typing",
@ -2491,14 +2499,6 @@
        "detail": "felicity.api.gql.types",
        "documentation": {}
    },
-    {
-        "label": "BytesScalar",
-        "importPath": "felicity.api.gql.types",
-        "description": "felicity.api.gql.types",
-        "isExtraImport": true,
-        "detail": "felicity.api.gql.types",
-        "documentation": {}
-    },
    {
        "label": "JSONScalar",
        "importPath": "felicity.api.gql.types",
@ -11841,6 +11841,38 @@
        "detail": "felicity.apps.iol.minio.enum",
        "documentation": {}
    },
+    {
+        "label": "MongoService",
+        "importPath": "felicity.database.mongo",
+        "description": "felicity.database.mongo",
+        "isExtraImport": true,
+        "detail": "felicity.database.mongo",
+        "documentation": {}
+    },
+    {
+        "label": "MongoCollection",
+        "importPath": "felicity.database.mongo",
+        "description": "felicity.database.mongo",
+        "isExtraImport": true,
+        "detail": "felicity.database.mongo",
+        "documentation": {}
+    },
+    {
+        "label": "MongoService",
+        "importPath": "felicity.database.mongo",
+        "description": "felicity.database.mongo",
+        "isExtraImport": true,
+        "detail": "felicity.database.mongo",
+        "documentation": {}
+    },
+    {
+        "label": "MongoCollection",
+        "importPath": "felicity.database.mongo",
+        "description": "felicity.database.mongo",
+        "isExtraImport": true,
+        "detail": "felicity.database.mongo",
+        "documentation": {}
+    },
    {
        "label": "schemas",
        "importPath": "felicity.apps.instrument",
@ -17820,6 +17852,14 @@
        "detail": "enum",
        "documentation": {}
    },
+    {
+        "label": "StrEnum",
+        "importPath": "enum",
+        "description": "enum",
+        "isExtraImport": true,
+        "detail": "enum",
+        "documentation": {}
+    },
    {
        "label": "BaseAuditModel",
        "importPath": "felicity.apps.common.schemas",
@ -21603,6 +21643,22 @@
        "detail": "sqlalchemy.orm.util",
        "documentation": {}
    },
+    {
+        "label": "ObjectId",
+        "importPath": "bson",
+        "description": "bson",
+        "isExtraImport": true,
+        "detail": "bson",
+        "documentation": {}
+    },
+    {
+        "label": "AsyncIOMotorClient",
+        "importPath": "motor.motor_asyncio",
+        "description": "motor.motor_asyncio",
+        "isExtraImport": true,
+        "detail": "motor.motor_asyncio",
+        "documentation": {}
+    },
    {
        "label": "weakref",
        "kind": 6,
@ -23668,7 +23724,7 @@
        "kind": 6,
        "importPath": "felicity.api.gql.impress.types",
        "description": "felicity.api.gql.impress.types",
-        "peekOfCode": "class ReportImpressType:\n    uid: str\n    state: str | None = None\n    sample_uid: str | None = None\n    sample: Optional[SampleType]\n    json_content: Optional[JSONScalar] = None\n    pdf_content: Optional[BytesScalar] = None\n    email_required: bool | None = None\n    email_sent: bool | None = None\n    sms_required: bool | None = None",
+        "peekOfCode": "class ReportImpressType:\n    uid: str\n    state: str | None = None\n    sample_uid: str | None = None\n    sample: Optional[SampleType]\n    # json_content: Optional[JSONScalar] = None\n    # pdf_content: Optional[BytesScalar] = None\n    email_required: bool | None = None\n    email_sent: bool | None = None\n    sms_required: bool | None = None",
        "detail": "felicity.api.gql.impress.types",
        "documentation": {}
    },
@ -35822,6 +35878,33 @@
        "detail": "felicity.database.queryset.builder",
        "documentation": {}
    },
+    {
+        "label": "MongoCollection",
+        "kind": 6,
+        "importPath": "felicity.database.mongo",
+        "description": "felicity.database.mongo",
+        "peekOfCode": "class MongoCollection(StrEnum):\n    DIAGNOSTIC_REPORT = \"diagnostic-report\"\n    INVOICE = \"invoice\"\nclass MongoService:\n    def __init__(self):\n        self.db = client.felicity\n    async def create(self, collection: MongoCollection, data: dict) -> Optional[dict]:\n        collection = self.db.get_collection(collection)\n        created = await collection.insert_one(data)\n        return await collection.find_one({\"_id\": created.inserted_id})",
+        "detail": "felicity.database.mongo",
+        "documentation": {}
+    },
+    {
+        "label": "MongoService",
+        "kind": 6,
+        "importPath": "felicity.database.mongo",
+        "description": "felicity.database.mongo",
+        "peekOfCode": "class MongoService:\n    def __init__(self):\n        self.db = client.felicity\n    async def create(self, collection: MongoCollection, data: dict) -> Optional[dict]:\n        collection = self.db.get_collection(collection)\n        created = await collection.insert_one(data)\n        return await collection.find_one({\"_id\": created.inserted_id})\n    async def upsert(self, collection: MongoCollection, uid: str, data: dict) -> Optional[dict]:\n        collection = self.db.get_collection(collection)\n        result = await collection.update_one({'_id': self.oid(uid)}, {'$set': data}, upsert=True)",
+        "detail": "felicity.database.mongo",
+        "documentation": {}
+    },
+    {
+        "label": "client",
+        "kind": 5,
+        "importPath": "felicity.database.mongo",
+        "description": "felicity.database.mongo",
+        "peekOfCode": "client = AsyncIOMotorClient(\"mongodb://felicity:felicity@localhost:27027\")\nclass MongoCollection(StrEnum):\n    DIAGNOSTIC_REPORT = \"diagnostic-report\"\n    INVOICE = \"invoice\"\nclass MongoService:\n    def __init__(self):\n        self.db = client.felicity\n    async def create(self, collection: MongoCollection, data: dict) -> Optional[dict]:\n        collection = self.db.get_collection(collection)\n        created = await collection.insert_one(data)",
+        "detail": "felicity.database.mongo",
+        "documentation": {}
+    },
    {
        "label": "EdgeNode",
        "kind": 6,
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -51,6 +51,21 @@ services:
      - "host.docker.internal:host-gateway"
    command: bash -c "cd felicity && alembic upgrade head && uvicorn main:felicity --host=0.0.0.0 --port=8000"

+  felicity_min:
+    container_name: felicity_min
+    image: bitnami/minio:2024.7.31
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    networks:
+      - felicitynet
+    volumes:
+      - 'minio_data:/data'
+    environment:
+      - MINIO_ROOT_USER=felicity
+      - MINIO_ROOT_PASSWORD=felicity
+      - MINIO_DEFAULT_BUCKET=felicity
+
  felicity_pg:
    container_name: felicity_pg
    image: postgres:12
@ -70,58 +85,53 @@ services:
    networks:
      - felicitynet

-  pgadmin:
-    container_name: felicity.pgadmin
-    image: dpage/pgadmin4
-    environment:
-      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@pgadmin.org}
-      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin}
-    volumes:
-      - pgadmin:/root/.pgadmin
-    ports:
-      - "${PGADMIN_PORT:-5050}:80"
-    networks:
-      - felicitynet
-    restart: unless-stopped
-    depends_on:
-      - felicity_pg
-
-  minio:
-    image: bitnami/minio:2024.7.31
-    ports:
-      - '9000:9000'
-      - '9001:9001'
-    networks:
-      - felicitynet
-    volumes:
-      - 'minio_data:/data'
-    environment:
-      - MINIO_ROOT_USER=felicity
-      - MINIO_ROOT_PASSWORD=felicity
-      - MINIO_DEFAULT_BUCKET=felicity
-
-  mongo:
+  felicity_mg:
+    container_name: felicity_mg
    image: mongo
    restart: always
    environment:
      MONGO_INITDB_ROOT_USERNAME: felicity
      MONGO_INITDB_ROOT_PASSWORD: felicity
+    ports:
+      - '27027:27017'
+    networks:
+      - felicitynet

-  mongo-express:
-    image: mongo-express
+  felicity_dg:
+    container_name: felicity_dg
+    image: dbgate/dbgate
    restart: always
    ports:
-      - 8081:8081
+      - 8050:3000
+    volumes:
+      - dbgate-data:/root/.dbgate
    environment:
-      ME_CONFIG_MONGODB_ADMINUSERNAME: felicity
-      ME_CONFIG_MONGODB_ADMINPASSWORD: felicity
-      ME_CONFIG_MONGODB_URL: mongodb://felicity:felicity@mongo:27017/
-      ME_CONFIG_BASICAUTH: false
+      LOGIN: felicity
+      PASSWORD: felicity
+      CONNECTIONS: pg,mg
+      LABEL_pg: Felicity Postgres
+      SERVER_pg: felicity_pg
+      USER_pg: felicity
+      PASSWORD_pg: felicity
+      DATABASE_pg: felicity_lims
+      PORT_pg: 5432
+      ENGINE_pg: postgres@dbgate-plugin-postgres
+      READONLY_pg: 1
+      # PASSWORD_MODE_pg: askUser
+      LABEL_mg: Felicity Mongo
+      URL_mg: mongodb://felicity:felicity@felicity_mg:27017/
+      DATABASE_mg: felicity
+      ENGINE_mg: mongo@dbgate-plugin-mongo
+      READONLY_mg: 1
+    networks:
+      - felicitynet

 volumes:
  pg-data:
  pgadmin:
  minio_data:
+  dbgate-data:
+    driver: local

 networks:
  felicitynet:
--- a/felicity/api/gql/impress/types.py
+++ b/felicity/api/gql/impress/types.py
@ -4,8 +4,9 @@ from typing import Optional
 import strawberry  # noqa

 from felicity.api.gql.analysis.types.analysis import SampleType
-from felicity.api.gql.types import BytesScalar, JSONScalar
+from felicity.api.gql.types import JSONScalar
 from felicity.api.gql.user.types import UserType
+from felicity.database.mongo import MongoService, MongoCollection


@strawberry.type
@ -14,8 +15,8 @@ class ReportImpressType:
    state: str | None = None
    sample_uid: str | None = None
    sample: Optional[SampleType]
-    json_content: Optional[JSONScalar] = None
-    pdf_content: Optional[BytesScalar] = None
+    # json_content: Optional[JSONScalar] = None
+    # pdf_content: Optional[BytesScalar] = None
    email_required: bool | None = None
    email_sent: bool | None = None
    sms_required: bool | None = None
@ -27,3 +28,7 @@ class ReportImpressType:
    created_at: datetime
    updated_by_uid: str | None = None
    updated_by: UserType | None = None
+
+    @strawberry.field
+    async def json_content(self) -> Optional[JSONScalar]:
+        return await MongoService().retrieve(MongoCollection.DIAGNOSTIC_REPORT, self.uid)
--- a/felicity/apps/impress/sample/utils.py
+++ b/felicity/apps/impress/sample/utils.py
@ -11,6 +11,7 @@ from felicity.apps.iol.minio.client import MinioClient
 from felicity.apps.iol.minio.enum import MinioBucket
 from felicity.apps.notification.services import ActivityStreamService
 from felicity.apps.setup.caches import get_laboratory
+from felicity.database.mongo import MongoService, MongoCollection
 from felicity.utils import remove_circular_refs

 logging.basicConfig(level=logging.INFO)
@ -64,7 +65,7 @@ async def impress_samples(sample_meta: List[any], user):
                **{
                    "state": report_state,
                    "sample_uid": sample.uid,
-                    "json_content": impress_meta,
+                    # "json_content": impress_meta,
                    # "pdf_content": sample_pdf,
                    "email_required": False,
                    "email_sent": False,
@ -73,8 +74,9 @@ async def impress_samples(sample_meta: List[any], user):
                    "generated_by_uid": user.uid,
                }
            )
-            await ReportImpressService().create(sc_in)
-            # save pdf to external storage
+            report_impress = await ReportImpressService().create(sc_in)
+
+            # save pdf to minio
            MinioClient().put_object(
                bucket=MinioBucket.DIAGNOSTIC_REPORT,
                object_name=sample.sample_id,
@ -82,9 +84,17 @@ async def impress_samples(sample_meta: List[any], user):
                metadata={
                    "state": report_state,
                    "sample_uid": sample.uid,
+                    "impress_meta_uid": report_impress.uid
                }
            )

+            # Save the json to mongodb
+            await MongoService().upsert(
+                collection=MongoCollection.DIAGNOSTIC_REPORT,
+                uid=report_impress.uid,
+                data=impress_meta
+            )
+
            if action != "pre-publish":
                sample = await SampleService().publish(sample.uid, published_by=user)

--- a/felicity/database/mongo.py
+++ b/felicity/database/mongo.py
@ -0,0 +1,61 @@
+from enum import StrEnum
+from typing import Optional
+
+from bson import ObjectId
+from motor.motor_asyncio import AsyncIOMotorClient
+
+client = AsyncIOMotorClient("mongodb://felicity:felicity@localhost:27027")
+
+
+class MongoCollection(StrEnum):
+    DIAGNOSTIC_REPORT = "diagnostic-report"
+    INVOICE = "invoice"
+
+
+class MongoService:
+    def __init__(self):
+        self.db = client.felicity
+
+    async def create(self, collection: MongoCollection, data: dict) -> Optional[dict]:
+        collection = self.db.get_collection(collection)
+        created = await collection.insert_one(data)
+        return await collection.find_one({"_id": created.inserted_id})
+
+    async def upsert(self, collection: MongoCollection, uid: str, data: dict) -> Optional[dict]:
+        collection = self.db.get_collection(collection)
+        result = await collection.update_one({'_id': self.oid(uid)}, {'$set': data}, upsert=True)
+        return await collection.find_one({"_id": result.upserted_id})
+
+    async def retrieve(self, collection: MongoCollection, uid: str):
+        collection = self.db.get_collection(collection)
+        item = await collection.find_one({"_id": self.oid(uid)})
+        item['_id'] = self.flake_id_from_hex(str(item['_id']))
+        return item
+
+    async def update(self, collection: MongoCollection, uid: str, data: dict) -> Optional[bool]:
+        collection = self.db.get_collection(collection)
+        if len(data) < 1:
+            return None
+        item = await collection.find_one({"_id": self.oid(uid)})
+        if item:
+            updated = await collection.update_one(
+                {"_id": self.oid(uid)}, {"$set": data}
+            )
+            return updated.matched_count > 0
+        return False
+
+    async def delete(self, collection: MongoCollection, uid: str) -> bool:
+        collection = self.db.get_collection(collection)
+        item = await collection.find_one({"_id": self.oid(uid)})
+        if item:
+            await collection.delete_one({"_id": self.oid(uid)})
+            return True
+        return False
+
+    @staticmethod
+    def oid(flake_id: str) -> ObjectId:
+        return ObjectId(hex(int(flake_id))[2:].zfill(24))
+
+    @staticmethod
+    def flake_id_from_hex(hex_string: str) -> str:
+        return str(int(hex_string, 16))
--- a/requirements.txt
+++ b/requirements.txt
@ -39,4 +39,5 @@ opentelemetry-instrumentation-sqlalchemy==0.43b0
 opentelemetry-instrumentation-psycopg2==0.43b0
 typing-extensions==4.11.0
 async-cache==1.1.1
-minio==7.2.7
+minio==7.2.7
+motor==3.5.1