diff --git a/DataSpace/asgi.py b/DataSpace/asgi.py index d3cfa0c4..dc791a8e 100644 --- a/DataSpace/asgi.py +++ b/DataSpace/asgi.py @@ -14,8 +14,9 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "DataSpace.settings") # Initialize OpenTelemetry before application -from api.telemetry import setup_telemetry +if os.getenv("TELEMETRY_URL"): + from api.telemetry import setup_telemetry -setup_telemetry() # Initialize telemetry for ASGI application + setup_telemetry() # Initialize telemetry for ASGI application application = get_asgi_application() diff --git a/DataSpace/wsgi.py b/DataSpace/wsgi.py index 0fa66c64..4fd7d875 100644 --- a/DataSpace/wsgi.py +++ b/DataSpace/wsgi.py @@ -12,8 +12,9 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "DataSpace.settings") # Initialize OpenTelemetry before application -from api.telemetry import setup_telemetry +if os.getenv("TELEMETRY_URL"): + from api.telemetry import setup_telemetry -setup_telemetry() # Initialize telemetry for production application + setup_telemetry() # Initialize telemetry for production application application = get_wsgi_application() diff --git a/Dockerfile b/Dockerfile index 19a08823..e3baf67d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,5 +60,5 @@ EXPOSE 8000 # Make entrypoint script executable RUN chmod +x /code/docker-entrypoint.sh -ENTRYPOINT ["/code/docker-entrypoint.sh"] +ENTRYPOINT ["bash","/code/docker-entrypoint.sh"] CMD ["uvicorn", "DataSpace.asgi:application", "--host", "0.0.0.0", "--port", "8000"] diff --git a/api/admin.py b/api/admin.py index 3ce529ea..d279d631 100644 --- a/api/admin.py +++ b/api/admin.py @@ -2,16 +2,45 @@ from api.models import ( AIModel, + AIModelVersion, Catalog, + Collaborative, + CollaborativeMetadata, + CollaborativeOrganizationRelationship, Dataset, + DatasetMetadata, + DataSpace, + Geography, + Metadata, ModelAPIKey, ModelEndpoint, Organization, + PromptDataset, + PromptResource, + Resource, + ResourceChartDetails, + ResourceChartImage, + ResourceDataTable, + ResourceFileDetails, + ResourceMetadata, + ResourcePreviewDetails, + ResourceSchema, + ResourceVersion, + SDG, + Sector, + Tag, UseCase, + UseCaseDashboard, + UseCaseMetadata, + UseCaseOrganizationRelationship, + VersionProvider, ) -# Register models needed for authorization app's autocomplete fields +# --------------------------------------------------------------------------- +# Organization +# --------------------------------------------------------------------------- + @admin.register(Organization) class OrganizationAdmin(admin.ModelAdmin): list_display = ("name", "slug", "created") @@ -19,6 +48,16 @@ class OrganizationAdmin(admin.ModelAdmin): prepopulated_fields = {"slug": ("name",)} +# --------------------------------------------------------------------------- +# Dataset & related +# --------------------------------------------------------------------------- + +@admin.register(Tag) +class TagAdmin(admin.ModelAdmin): + list_display = ("value",) + search_fields = ("value",) + + @admin.register(Dataset) class DatasetAdmin(admin.ModelAdmin): list_display = ("title", "organization", "created") @@ -26,6 +65,24 @@ class DatasetAdmin(admin.ModelAdmin): search_fields = ("title", "description") +@admin.register(DatasetMetadata) +class DatasetMetadataAdmin(admin.ModelAdmin): + list_display = ("dataset", "metadata_item", "value") + list_filter = ("dataset",) + search_fields = ("value",) + + +@admin.register(PromptDataset) +class PromptDatasetAdmin(admin.ModelAdmin): + list_display = ("title", "task_type", "domain", "purpose", "organization", "created") + list_filter = ("task_type", "domain", "purpose", "organization") + search_fields = ("title", "description") + + +# --------------------------------------------------------------------------- +# UseCase & related +# --------------------------------------------------------------------------- + @admin.register(UseCase) class UseCaseAdmin(admin.ModelAdmin): list_display = ("title", "slug", "created") @@ -33,11 +90,139 @@ class UseCaseAdmin(admin.ModelAdmin): list_filter = ("organization",) +@admin.register(UseCaseDashboard) +class UseCaseDashboardAdmin(admin.ModelAdmin): + list_display = ("usecase", "name") + search_fields = ("usecase__title", "name") + + +@admin.register(UseCaseMetadata) +class UseCaseMetadataAdmin(admin.ModelAdmin): + list_display = ("usecase", "metadata_item", "value") + list_filter = ("usecase",) + search_fields = ("value",) + + +@admin.register(UseCaseOrganizationRelationship) +class UseCaseOrganizationRelationshipAdmin(admin.ModelAdmin): + list_display = ("usecase", "organization", "relationship_type") + list_filter = ("relationship_type",) + search_fields = ("usecase__title", "organization__name") + + +# --------------------------------------------------------------------------- +# Catalog +# --------------------------------------------------------------------------- + @admin.register(Catalog) class CatalogAdmin(admin.ModelAdmin): list_display = ("name", "slug", "created") +# --------------------------------------------------------------------------- +# Collaborative & related +# --------------------------------------------------------------------------- + +@admin.register(Collaborative) +class CollaborativeAdmin(admin.ModelAdmin): + list_display = ("title", "slug", "created") + search_fields = ("title", "slug", "description") + prepopulated_fields = {"slug": ("title",)} + + +@admin.register(CollaborativeMetadata) +class CollaborativeMetadataAdmin(admin.ModelAdmin): + list_display = ("collaborative", "metadata_item", "value") + list_filter = ("collaborative",) + search_fields = ("value",) + + +@admin.register(CollaborativeOrganizationRelationship) +class CollaborativeOrganizationRelationshipAdmin(admin.ModelAdmin): + list_display = ("collaborative", "organization", "relationship_type") + list_filter = ("relationship_type",) + search_fields = ("collaborative__title", "organization__name") + + +# --------------------------------------------------------------------------- +# Resource & related +# --------------------------------------------------------------------------- + +class ResourceFileDetailsInline(admin.StackedInline): + model = ResourceFileDetails + extra = 0 + + +@admin.register(Resource) +class ResourceAdmin(admin.ModelAdmin): + list_display = ("name", "dataset", "type", "created") + list_filter = ("type", "dataset") + search_fields = ("name", "description") + inlines = [ResourceFileDetailsInline] + + +@admin.register(ResourceFileDetails) +class ResourceFileDetailsAdmin(admin.ModelAdmin): + list_display = ("resource",) + search_fields = ("resource__title",) + + +@admin.register(ResourcePreviewDetails) +class ResourcePreviewDetailsAdmin(admin.ModelAdmin): + list_display = ("resource",) + search_fields = ("resource__title",) + + +@admin.register(ResourceDataTable) +class ResourceDataTableAdmin(admin.ModelAdmin): + list_display = ("resource",) + search_fields = ("resource__title",) + + +@admin.register(ResourceVersion) +class ResourceVersionAdmin(admin.ModelAdmin): + list_display = ("resource", "version_number", "created_at") + list_filter = ("resource",) + search_fields = ("resource__name", "version_number") + + +@admin.register(ResourceChartDetails) +class ResourceChartDetailsAdmin(admin.ModelAdmin): + list_display = ("resource",) + search_fields = ("resource__title",) + + +@admin.register(ResourceChartImage) +class ResourceChartImageAdmin(admin.ModelAdmin): + list_display = ("name", "dataset", "status") + list_filter = ("dataset", "status") + search_fields = ("name", "description") + + +@admin.register(ResourceMetadata) +class ResourceMetadataAdmin(admin.ModelAdmin): + list_display = ("resource", "metadata_item", "value") + list_filter = ("resource",) + search_fields = ("value",) + + +@admin.register(ResourceSchema) +class ResourceSchemaAdmin(admin.ModelAdmin): + list_display = ("resource",) + search_fields = ("resource__title",) + + +@admin.register(PromptResource) +class PromptResourceAdmin(admin.ModelAdmin): + list_display = ("resource", "prompt_format", "created") + list_filter = ("resource__dataset", "prompt_format") + search_fields = ("resource__name",) + + +# --------------------------------------------------------------------------- +# AIModel & related +# --------------------------------------------------------------------------- + class ModelEndpointInline(admin.TabularInline): model = ModelEndpoint extra = 1 @@ -110,6 +295,21 @@ class AIModelAdmin(admin.ModelAdmin): ) +@admin.register(AIModelVersion) +class AIModelVersionAdmin(admin.ModelAdmin): + list_display = ("ai_model", "version", "status", "created_at") + list_filter = ("status",) + search_fields = ("ai_model__name", "version") + readonly_fields = ("created_at", "updated_at") + + +@admin.register(VersionProvider) +class VersionProviderAdmin(admin.ModelAdmin): + list_display = ("version", "provider", "is_primary", "is_active") + list_filter = ("provider", "is_primary", "is_active") + search_fields = ("version__ai_model__name", "provider_model_id") + + @admin.register(ModelEndpoint) class ModelEndpointAdmin(admin.ModelAdmin): list_display = ( @@ -201,3 +401,47 @@ class ModelAPIKeyAdmin(admin.ModelAdmin): {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}, ), ) + + + +# --------------------------------------------------------------------------- +# Geography / SDG / Sector +# --------------------------------------------------------------------------- + +@admin.register(Geography) +class GeographyAdmin(admin.ModelAdmin): + list_display = ("name",) + search_fields = ("name",) + + +@admin.register(SDG) +class SDGAdmin(admin.ModelAdmin): + list_display = ("name", "number") + search_fields = ("name",) + + +@admin.register(Sector) +class SectorAdmin(admin.ModelAdmin): + list_display = ("name",) + search_fields = ("name",) + + +# --------------------------------------------------------------------------- +# DataSpace +# --------------------------------------------------------------------------- + +@admin.register(DataSpace) +class DataSpaceAdmin(admin.ModelAdmin): + list_display = ("name", "created") + search_fields = ("name",) + + +# --------------------------------------------------------------------------- +# Metadata +# --------------------------------------------------------------------------- + +@admin.register(Metadata) +class MetadataAdmin(admin.ModelAdmin): + list_display = ("label", "data_type", "type", "model", "enabled", "filterable", "created") + list_filter = ("data_type", "type", "model", "enabled", "filterable") + search_fields = ("label", "urn") diff --git a/api/migrations/0001_initial.py b/api/migrations/0001_initial.py old mode 100644 new mode 100755 diff --git a/api/migrations/__init__.py b/api/migrations/__init__.py old mode 100644 new mode 100755 diff --git a/api/models/Collaborative.py b/api/models/Collaborative.py index 6f3469b4..0681aa2b 100644 --- a/api/models/Collaborative.py +++ b/api/models/Collaborative.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Any, cast from django.db import models +from django.core.validators import RegexValidator from django.utils.text import slugify if TYPE_CHECKING: @@ -13,6 +14,12 @@ from api.utils.file_paths import _use_case_directory_path +slug_validator = RegexValidator( + regex=r"^[a-z0-9]+(?:-[a-z0-9]+)*$", + message="Slug must be lowercase and contain only alphanumeric characters and hyphens.", +) + + class Collaborative(models.Model): id = models.AutoField(primary_key=True) title = models.CharField(max_length=200, unique=True, blank=True, null=True) @@ -27,7 +34,9 @@ class Collaborative(models.Model): modified = models.DateTimeField(auto_now=True) website = models.URLField(blank=True) contact_email = models.EmailField(blank=True, null=True) - slug = models.SlugField(max_length=75, null=True, blank=True, unique=True) + slug = models.SlugField( + max_length=75, null=True, blank=True, unique=True, validators=[slug_validator] + ) user = models.ForeignKey("authorization.User", on_delete=models.CASCADE) organization = models.ForeignKey( "api.Organization", on_delete=models.CASCADE, null=True, blank=True @@ -62,8 +71,9 @@ class Collaborative(models.Model): platform_url = models.URLField(blank=True, null=True) def save(self, *args: Any, **kwargs: Any) -> None: - if self.title: + if self.title and not self.slug: self.slug = slugify(cast(str, self.title)) + self.full_clean() super().save(*args, **kwargs) @property diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index 3f0570cb..3c1213d7 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -78,6 +78,7 @@ class CollaborativeInputPartial: logo: Optional[Upload] = strawberry.field(default=None) cover_image: Optional[Upload] = strawberry.field(default=None) title: Optional[str] = None + slug: Optional[str] = None summary: Optional[str] = None platform_url: Optional[str] = None tags: Optional[List[str]] = None @@ -426,6 +427,10 @@ def update_collaborative( if data.title.strip() == "": raise ValueError("Title cannot be empty.") collaborative.title = data.title.strip() + if data.slug is not None: + if data.slug.strip() == "": + raise ValueError("Slug cannot be empty.") + collaborative.slug = data.slug.strip() if data.summary is not None: collaborative.summary = data.summary.strip() if data.platform_url is not None: diff --git a/docker-compose.yml b/docker-compose.yml index 607d99b4..e67e1ca1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.9" + services: backend: build: . @@ -111,6 +111,7 @@ services: image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2 container_name: telemetry_elasticsearch restart: always + profiles: ["telemetry"] ulimits: memlock: soft: -1 @@ -140,6 +141,7 @@ services: kibana: image: docker.elastic.co/kibana/kibana:7.16.2 container_name: kibana + profiles: ["telemetry"] environment: ELASTICSEARCH_URL: "http://telemetry_elasticsearch:9200" ELASTICSEARCH_HOSTS: '["http://telemetry_elasticsearch:9200"]' @@ -150,11 +152,12 @@ services: telemetry_elasticsearch: condition: service_healthy ports: - - 5601:5601 + - "5601:5601" apm-server: image: docker.elastic.co/apm/apm-server:7.16.2 container_name: apm-server + profiles: ["telemetry"] user: apm-server restart: always command: @@ -199,6 +202,7 @@ services: otel-collector: image: otel/opentelemetry-collector:latest container_name: otel-collector + profiles: ["telemetry"] restart: always command: "--config=/etc/otel-collector-config.yaml" volumes: @@ -207,7 +211,7 @@ services: apm-server: condition: service_healthy ports: - - 4317:4317 + - "4317:4317" logging: options: max-size: "10m" diff --git a/otel-collector-config.yaml b/otel-collector-config.yaml index 4d754075..d19dd5ec 100644 --- a/otel-collector-config.yaml +++ b/otel-collector-config.yaml @@ -10,8 +10,9 @@ processors: batch: exporters: - logging: - loglevel: warn + # 'logging' is replaced by 'debug' + debug: + verbosity: normal # 'normal' is equivalent to the old 'info'/'warn' levels otlp/elastic: endpoint: "apm-server:8200" tls: @@ -26,9 +27,9 @@ service: pipelines: traces: receivers: [otlp] - exporters: [logging, otlp/elastic] - processors: [batch] + processors: [memory_limiter, batch] # Added memory_limiter to the pipeline + exporters: [debug, otlp/elastic] metrics: receivers: [otlp] - exporters: [logging, otlp/elastic] - processors: [batch] + processors: [memory_limiter, batch] # Added memory_limiter to the pipeline + exporters: [debug, otlp/elastic] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 56a97311..aad4a741 100644 --- a/requirements.txt +++ b/requirements.txt @@ -112,7 +112,7 @@ django-stubs==4.2.7 djangorestframework-stubs==3.14.5 #whitenoise for managing static files - whitenoise==6.9.0 +whitenoise==6.9.0 # Activity stream for tracking user actions django-activity-stream==2.0.0