Skip to content

Commit 8836caa

Browse files
fedjofedjo
authored
Create a robust docker image build for ARMD64 and ARM64 (#81)
* Multiplatform builds - arm64 support (#2) * Add Dockerfile and compose file for arm64 * Rename docker files to x86 architecture * Add multi platform workflow with qemu * Typo * Sanitize branch name for docker image tag * Rename file * Try platform build in two jobs * Correct label tags * Create correct manifest * Debug caching in arm64 * Add category field when creating FC activity type, fix some typing errors (#10) * Add optional, hasAgriParcel field to FC Observation (#9) * Add optional, hasAgriParcel field to FC Observation * Change value type to dict * Fix pydantic inherit error (#11) * Run eagerly all build jobs and update manifest (#12) * Robust build in workflow and compose
1 parent 1fbe7cf commit 8836caa

9 files changed

Lines changed: 247 additions & 29 deletions

File tree

‎.github/workflows/docker-image.yml‎

Lines changed: 108 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Docker Image CI
1+
name: Docker Multi-Platform Builds
22

33
on:
44
push:
@@ -9,22 +9,21 @@ on:
99
description: 'Branch to run the workflow on'
1010
required: false
1111
release:
12-
types:
13-
- created
14-
12+
types: [ created ]
1513

1614
jobs:
17-
build-and-push:
15+
16+
# -------------------------
17+
# AMD64 Build
18+
# -------------------------
19+
build-amd64:
1820
runs-on: ubuntu-latest
1921
permissions:
2022
contents: read
2123
packages: write
22-
2324
steps:
2425
- name: Checkout code
2526
uses: actions/checkout@v4
26-
with:
27-
fetch-depth: 0 # Saves time by not fetching other branches
2827

2928
- name: Set up Docker Buildx
3029
uses: docker/setup-buildx-action@v3
@@ -36,18 +35,114 @@ jobs:
3635
username: ${{ github.repository_owner }}
3736
password: ${{ secrets.GITHUB_TOKEN }}
3837

39-
- name: Set lowercase repository name
38+
- name: Normalize repo and sanitize ref
4039
run: |
4140
echo "REPO_LOWER=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
41+
SAFE_REF=$(echo "${GITHUB_REF_NAME}" | tr / -)
42+
echo "SAFE_REF=$SAFE_REF" >> $GITHUB_ENV
4243
43-
- name: Build and push Docker image
44-
id: docker_build
44+
- name: Build and push AMD64 image
4545
uses: docker/build-push-action@v6
4646
with:
4747
context: .
48+
file: Dockerfile.x86_64
49+
build-args: SOURCE_REPO=https://github.com/${{ github.repository }}
50+
platforms: linux/amd64
4851
push: true
4952
tags: |
50-
ghcr.io/${{ env.REPO_LOWER }}:${{ github.ref_name }}
51-
${{ github.ref_name == 'main' && format('ghcr.io/{0}:latest', env.REPO_LOWER) || '' }}
53+
ghcr.io/${{ env.REPO_LOWER }}:${{ env.SAFE_REF }}-amd64
54+
${{ github.ref_name == 'main' && format('ghcr.io/{0}:latest-amd64', env.REPO_LOWER) || '' }}
5255
cache-from: type=gha
5356
cache-to: type=gha,mode=max
57+
58+
# -------------------------
59+
# ARM64 Build (manual / self-hosted)
60+
# -------------------------
61+
build-arm64:
62+
#if: ${{ github.event_name == 'workflow_dispatch' }} # manual trigger only
63+
# runs-on: [self-hosted, arm64] # use if you have an ARM64 runner
64+
runs-on: ubuntu-latest
65+
steps:
66+
- name: Checkout code
67+
uses: actions/checkout@v4
68+
69+
- name: Set up Docker daemon (containerd features)
70+
uses: docker/setup-docker-action@v4
71+
with:
72+
daemon-config: |
73+
{
74+
"debug": true,
75+
"features": { "containerd-snapshotter": true }
76+
}
77+
78+
- name: Set up QEMU
79+
uses: docker/setup-qemu-action@v3
80+
81+
- name: Set up Docker Buildx
82+
uses: docker/setup-buildx-action@v3
83+
84+
- name: Login to GitHub Container Registry
85+
uses: docker/login-action@v3
86+
with:
87+
registry: ghcr.io
88+
username: ${{ github.repository_owner }}
89+
password: ${{ secrets.GITHUB_TOKEN }}
90+
91+
- name: Normalize repo and sanitize ref
92+
run: |
93+
echo "REPO_LOWER=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
94+
SAFE_REF=$(echo "${GITHUB_REF_NAME}" | tr / -)
95+
echo "SAFE_REF=$SAFE_REF" >> $GITHUB_ENV
96+
97+
- name: Build and push ARM64 image
98+
uses: docker/build-push-action@v6
99+
with:
100+
context: .
101+
file: Dockerfile.arm64
102+
build-args: BUILDKIT_PROGRESS=plain,SOURCE_REPO=https://github.com/${{ github.repository }}
103+
platforms: linux/arm64
104+
push: true
105+
tags: |
106+
ghcr.io/${{ env.REPO_LOWER }}:${{ env.SAFE_REF }}-arm64
107+
${{ github.ref_name == 'main' && format('ghcr.io/{0}:latest-arm64', env.REPO_LOWER) || '' }}
108+
cache-from: type=gha,scope=arm64
109+
cache-to: type=gha,scope=arm64,mode=max
110+
111+
# -------------------------
112+
# Create multi-arch manifest
113+
# -------------------------
114+
manifest:
115+
# if: ${{ github.event_name == 'workflow_dispatch' }} # only when both images are built
116+
needs: [ build-amd64, build-arm64 ]
117+
runs-on: ubuntu-latest
118+
steps:
119+
- name: Set up Docker Buildx (for imagetools)
120+
uses: docker/setup-buildx-action@v3
121+
122+
- name: Login to GitHub Container Registry
123+
uses: docker/login-action@v3
124+
with:
125+
registry: ghcr.io
126+
username: ${{ github.repository_owner }}
127+
password: ${{ secrets.GITHUB_TOKEN }}
128+
129+
- name: Normalize repo and sanitize ref
130+
run: |
131+
echo "REPO_LOWER=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
132+
SAFE_REF=$(echo "${GITHUB_REF_NAME}" | tr / -)
133+
echo "SAFE_REF=$SAFE_REF" >> $GITHUB_ENV
134+
135+
- name: Create multi-arch manifest for ref tag
136+
run: |
137+
docker buildx imagetools create \
138+
--tag ghcr.io/${{ env.REPO_LOWER }}:${{ env.SAFE_REF }} \
139+
ghcr.io/${{ env.REPO_LOWER }}:${{ env.SAFE_REF }}-amd64 \
140+
ghcr.io/${{ env.REPO_LOWER }}:${{ env.SAFE_REF }}-arm64
141+
142+
- name: Also tag :latest on main
143+
if: ${{ github.ref_name == 'main' }}
144+
run: |
145+
docker buildx imagetools create \
146+
--tag ghcr.io/${{ env.REPO_LOWER }}:latest \
147+
ghcr.io/${{ env.REPO_LOWER }}:latest-amd64 \
148+
ghcr.io/${{ env.REPO_LOWER }}:latest-arm64

Dockerfile renamed to Dockerfile.arm64

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
ARG SOURCE_REPO=https://github.com/openagri-eu/openagri-weatherservice
2+
13
FROM python:3.12 AS builder
24

3-
LABEL org.opencontainers.image.source https://github.com/agstack/weather-service
5+
LABEL org.opencontainers.image.source=${SOURCE_REPO}
46

57
ARG DEBIAN_FRONTEND=noninteractive
68

@@ -43,4 +45,5 @@ ENV USER_ID=$USER_ID \
4345
PROJECT_DIR=/weather-service \
4446
PYTHONPATH=/weather-service
4547

46-
CMD ["./run.sh", "prod"]
48+
ENTRYPOINT ["./run.sh"]
49+
CMD ["prod"]

‎Dockerfile.x86_64‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Define the Build Argument
2+
ARG SOURCE_REPO=https://github.com/openagri-eu/openagri-weatherservice
3+
4+
FROM python:3.12 AS builder
5+
6+
LABEL org.opencontainers.image.source=${SOURCE_REPO}
7+
8+
ARG DEBIAN_FRONTEND=noninteractive
9+
10+
RUN set -x && \
11+
apt-get update -q && \
12+
apt-get install -yq --no-install-recommends && \
13+
apt-get autoremove -yq && \
14+
apt-get clean -q && rm -rf /var/lib/apt/lists/* && \
15+
find / -name '*.py[co]' -delete
16+
17+
COPY requirements.txt requirements-test.txt ./
18+
19+
RUN python3 -m pip install --upgrade pip && \
20+
python3 -m pip install -U --no-cache-dir -r requirements.txt && \
21+
find / -name '*.py[co]' -delete
22+
23+
FROM python:3.12-slim AS runner
24+
25+
26+
WORKDIR /weather-service
27+
28+
COPY --from=builder /usr/local/lib/python3.12/site-packages/ /usr/local/lib/python3.12/site-packages/
29+
COPY --from=builder /usr/local/bin/ /usr/local/bin/
30+
31+
ARG USER_ID=1001
32+
ARG GROUP_ID=1001
33+
RUN groupadd -r openagri --gid $GROUP_ID && \
34+
useradd -d /home/openagri -ms /bin/bash -r -g openagri openagri --uid $USER_ID
35+
36+
COPY src ./src
37+
# COPY tests ./tests
38+
COPY run.sh ./
39+
40+
RUN chown -R $USER_ID:$GROUP_ID /weather-service
41+
42+
USER openagri
43+
44+
ENV USER_ID=$USER_ID \
45+
GROUP_ID=$GROUP_ID \
46+
PROJECT_DIR=/weather-service \
47+
PYTHONPATH=/weather-service
48+
49+
ENTRYPOINT ["./run.sh"]
50+
CMD ["prod"]

‎docker-compose-arm64.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
version: '3'
2+
3+
4+
services:
5+
app:
6+
image: ghcr.io/${DOCKER_REGISTRY}/openagri-weatherservice:${TAG}
7+
build:
8+
context: .
9+
dockerfile: Dockerfile.arm64
10+
# Pass the build argument from your environment
11+
args:
12+
SOURCE_REPO: ${SOURCE_REPO}
13+
depends_on:
14+
- mongodb
15+
ports:
16+
- "${WEATHER_SRV_PORT}:${WEATHER_SRV_PORT}"
17+
environment:
18+
# Generic environment vars
19+
LOGGING_LEVEL: ${LOGGING_LEVEL:-INFO}
20+
JWT_KEY: ${JWT_SIGNING_KEY}
21+
ALGORITHM: ${JWT_ALG:-HS256}
22+
CRYPT_CONTEXT_SCHEME: ${JWT_CRYPT_CONTEXT_SCHEME:-bcrypt}
23+
ACCESS_TOKEN_EXPIRE_MINUTES: ${JWT_ACCESS_TOKEN_EXPIRATION_TIME:-240}
24+
# Service specific vars
25+
WEATHER_SRV_PORT: ${WEATHER_SRV_PORT:-8000}
26+
WEATHER_SRV_HOSTNAME: ${WEATHER_SRV_HOSTNAME:-weathersrv}
27+
WEATHER_SRV_DATABASE_URI: ${WEATHER_SRV_DATABASE_URI}
28+
WEATHER_SRV_DATABASE_NAME: ${WEATHER_SRV_DATABASE_NAME}
29+
EXTRA_ALLOWED_HOSTS: ${WEATHER_SRV_EXTRA_ALLOWED_HOSTS}
30+
WEATHER_SRV_OPENWEATHERMAP_API_KEY: ${WEATHER_SRV_OPENWEATHERMAP_API_KEY}
31+
# If the below value is set then weather service will try to integrate with Gatekeeper
32+
GATEKEEPER_URL: ${INTERNAL_GK_URL}
33+
WEATHER_SRV_GATEKEEPER_USER: ${GATEKEEPER_SUPERUSER_USERNAME}
34+
WEATHER_SRV_GATEKEEPER_PASSWORD: ${GATEKEEPER_SUPERUSER_PASSWORD}
35+
# FARM CALENDAR
36+
PUSH_THI_TO_FARMCALENDAR: ${PUSH_THI_TO_FARMCALENDAR}
37+
GATEKEEPER_FARM_CALENDAR_API: ${GATEKEEPER_FARM_CALENDAR_API}
38+
INTERVAL_THI_TO_FARMCALENDAR: ${INTERVAL_THI_TO_FARMCALENDAR}
39+
volumes:
40+
- ./data:/data:rw
41+
42+
mongodb:
43+
image: arm64v8/mongo:4.4.18
44+
command: mongod
45+
ports:
46+
- '24252:24252'
47+
- '27017:27017'
48+
environment:
49+
MONGO_INITDB_ROOT_USERNAME: ${WEATHER_SRV_MONGO_INITDB_ROOT_USERNAME}
50+
MONGO_INITDB_ROOT_PASSWORD: ${WEATHER_SRV_MONGO_INITDB_ROOT_PASSWORD}
51+
MONGO_INITDB_DATABASE: ${WEATHER_SRV_MONGO_INITDB_DATABASE}
52+
volumes:
53+
- mongo_data:/data/db
54+
55+
volumes:
56+
mongo_data:

docker-compose.yml renamed to docker-compose-x86_64.yml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
services:
22
app:
3-
image: ghcr.io/openagri-eu/openagri-weatherservice:${TAG}
4-
build: .
3+
image: ghcr.io/${DOCKER_REGISTRY}/openagri-weatherservice:${TAG}
4+
build:
5+
context: .
6+
dockerfile: Dockerfile.x86_64
7+
# Pass the build argument from your environment
8+
args:
9+
SOURCE_REPO: ${SOURCE_REPO}
510
depends_on:
611
- mongodb
712
ports:

‎env.example‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
# Version
1+
# Build
22
TAG=latest
3+
DOCKER_REGISTRY=openagri-eu
4+
SOURCE_REPO=https://github.com/openagri-eu/openagri-weatherservice
5+
36

47
# Weather service
58
LOGGING_LEVEL=DEBUG

‎run.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ export PYTHONPATH=$PYTHONPATH:$(pwd)
77
run_uvicorn() {
88
set -x
99
# Uvicorn (ASGI)
10-
uvicorn --host 0.0.0.0 --port $WEATHER_SRV_PORT 'src.main:create_app'
10+
exec uvicorn --host 0.0.0.0 --port $WEATHER_SRV_PORT 'src.main:create_app'
1111
}
1212

1313
unittest() {

‎src/openagri_services/farmcalendar_service.py‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1+
import logging
12
import asyncio
23
import json
34
import re
45
import time
56
from typing import Optional, Tuple
67
from uuid import uuid4
7-
from fastapi import FastAPI, HTTPException
8-
import logging
98

9+
from fastapi import FastAPI, HTTPException
1010
import backoff
1111

1212
from src.core import config
@@ -31,13 +31,14 @@ def __init__(self, app: FastAPI):
3131
on_backoff=lambda details: asyncio.create_task(details['args'][0].app.setup_authentication_tokens()),
3232
max_tries=3
3333
)
34-
async def fetch_or_create_activity_type(self, activity_type: str, description: str) -> str:
34+
async def fetch_or_create_activity_type(self, activity_type: str, description: str, category="observation") -> str:
3535
act_jsonld = await self.get(f'/FarmCalendarActivityTypes/?name={activity_type}')
3636

3737
if not self._get_activity_type_id(act_jsonld):
3838
json_payload = {
3939
"name": activity_type,
4040
"description": description,
41+
"category": category
4142
}
4243
act_jsonld = await self.post('/FarmCalendarActivityTypes/', json=json_payload)
4344

@@ -49,26 +50,26 @@ async def fetch_or_create_thi_activity_type(self) -> str:
4950
'THI_Observation',
5051
'Activity type collecting observed values for Temperature Humidity Index'
5152
)
53+
return self.thi_activity_type
5254

5355
# Create Flight Forecast Observation Activity Type
5456
async def fetch_or_create_flight_forecast_activity_type(self) -> str:
5557
self.ff_activity_type = await self.fetch_or_create_activity_type(
5658
'Flight_Forecast_Observation',
5759
'Activity type collecting observed values for UAV Flight Forecast'
5860
)
61+
return self.ff_activity_type
5962

6063
# Create spray conditions forecast Observation Activity Type
6164
async def fetch_or_create_spray_forecast_activity_type(self) -> str:
6265
self.sp_activity_type = await self.fetch_or_create_activity_type(
6366
'Spray_Forecast_Observation',
6467
'Activity type collectins observed values for spray conditions forecast'
6568
)
69+
return self.sp_activity_type
6670

67-
def _get_activity_type_id(self, jsonld: dict) -> Optional[str]:
68-
if jsonld['@graph']:
69-
return jsonld["@graph"][0]["@id"]
70-
return
71-
71+
def _get_activity_type_id(self, jsonld: dict) -> str:
72+
return jsonld["@graph"][0]["@id"]
7273

7374
# Fetch locations from FARM_CALENDAR_URI
7475
@backoff.on_exception(

0 commit comments

Comments
 (0)