Docker build inside serverless

MBalk · 2024-05-09T13:59:36.373Z

Hey all, I am pretty new to serverless, python and docker. I am trying to build an image that I want to run on runpod, but I am running into the issues that some of the dependencies cannot be build on my macbook pro or github actions. So I thought, why not build the image on runpod as well. I created the following `handler.py` (See message below) But receive the below error. I think it is because docker is not running within the container. Is there a way to make this work? `Dockerfile` ``` FROM runpod/base:0.4.0-cuda11.8.0 COPY builder/requirements.txt /requirements.txt RUN python3.11 -m pip install --upgrade pip && \ python3.11 -m pip install --upgrade -r /requirements.txt --no-cache-dir && \ rm /requirements.txt ADD src . CMD python3.11 -u /handler.py ```

M

MBalkOP•5/9/24, 1:59 PM

import runpod

import os
import git
import docker

def clone_repo(repo_url, repo_name):
    # Clone the GitHub repository
    git.Repo.clone_from(repo_url, repo_name)

def build_docker_image(dockerfile_path, image_name, image_tag):
    # Initialize Docker client
    client = docker.DockerClient(base_url='unix://var/run/docker.sock')

    # Build the Docker image
    image, build_logs = client.images.build(
        path=os.path.dirname(dockerfile_path),
        dockerfile=os.path.basename(dockerfile_path),
        tag=f"{image_name}:{image_tag}",
    )

    # Print build logs
    for log in build_logs:
        print(log)

def push_docker_image(username, image_name, image_tag, github_token):
    # Initialize Docker client
    client = docker.DockerClient(base_url='unix://var/run/docker.sock')

    # Authenticate with GitHub Docker Registry using access token
    client.login(username=username, password=github_token, registry="ghcr.io")

    # Push the Docker image to GitHub Docker Registry
    client.images.push(repository=f"ghcr.io/{image_name}", tag=image_tag)

def handler(job):
    job_input = job["input"]
    
    # GitHub access token
    github_token = os.environ["GITHUB_TOKEN"]

    # GitHub repository information
    username = job_input["username"]
    repo_name = job_input["repo_name"]
    repo_url = f"https://{github_token}:x-oauth-basic@github.com/{username}/{repo_name}.git"

    # Docker image information
    dockerfile_path = f"./{repo_name}/Dockerfile"
    image_name = f"{username}/{repo_name}"
    image_tag = job_input.get("image_tag", "latest")

    # Clone the GitHub repository
    clone_repo(repo_url, repo_name)

    # Build the Docker image
    build_docker_image(dockerfile_path, image_name, image_tag)

    # Push the Docker image to GitHub Docker Registry
    result = push_docker_image(username, image_name, image_tag, github_token)
    
    return result
    
runpod.serverless.start({"handler": handler})

import runpod

import os
import git
import docker

def clone_repo(repo_url, repo_name):
    # Clone the GitHub repository
    git.Repo.clone_from(repo_url, repo_name)

def build_docker_image(dockerfile_path, image_name, image_tag):
    # Initialize Docker client
    client = docker.DockerClient(base_url='unix://var/run/docker.sock')

    # Build the Docker image
    image, build_logs = client.images.build(
        path=os.path.dirname(dockerfile_path),
        dockerfile=os.path.basename(dockerfile_path),
        tag=f"{image_name}:{image_tag}",
    )

    # Print build logs
    for log in build_logs:
        print(log)

def push_docker_image(username, image_name, image_tag, github_token):
    # Initialize Docker client
    client = docker.DockerClient(base_url='unix://var/run/docker.sock')

    # Authenticate with GitHub Docker Registry using access token
    client.login(username=username, password=github_token, registry="ghcr.io")

    # Push the Docker image to GitHub Docker Registry
    client.images.push(repository=f"ghcr.io/{image_name}", tag=image_tag)

def handler(job):
    job_input = job["input"]
    
    # GitHub access token
    github_token = os.environ["GITHUB_TOKEN"]

    # GitHub repository information
    username = job_input["username"]
    repo_name = job_input["repo_name"]
    repo_url = f"https://{github_token}:x-oauth-basic@github.com/{username}/{repo_name}.git"

    # Docker image information
    dockerfile_path = f"./{repo_name}/Dockerfile"
    image_name = f"{username}/{repo_name}"
    image_tag = job_input.get("image_tag", "latest")

    # Clone the GitHub repository
    clone_repo(repo_url, repo_name)

    # Build the Docker image
    build_docker_image(dockerfile_path, image_name, image_tag)

    # Push the Docker image to GitHub Docker Registry
    result = push_docker_image(username, image_name, image_tag, github_token)
    
    return result
    
runpod.serverless.start({"handler": handler})

import runpod

import os
import git
import docker

def clone_repo(repo_url, repo_name):
    # Clone the GitHub repository
    git.Repo.clone_from(repo_url, repo_name)

def build_docker_image(dockerfile_path, image_name, image_tag):
    # Initialize Docker client
    client = docker.DockerClient(base_url='unix://var/run/docker.sock')

    # Build the Docker image
    image, build_logs = client.images.build(
        path=os.path.dirname(dockerfile_path),
        dockerfile=os.path.basename(dockerfile_path),
        tag=f"{image_name}:{image_tag}",
    )

    # Print build logs
    for log in build_logs:
        print(log)

def push_docker_image(username, image_name, image_tag, github_token):
    # Initialize Docker client
    client = docker.DockerClient(base_url='unix://var/run/docker.sock')

    # Authenticate with GitHub Docker Registry using access token
    client.login(username=username, password=github_token, registry="ghcr.io")

    # Push the Docker image to GitHub Docker Registry
    client.images.push(repository=f"ghcr.io/{image_name}", tag=image_tag)

def handler(job):
    job_input = job["input"]
    
    # GitHub access token
    github_token = os.environ["GITHUB_TOKEN"]

    # GitHub repository information
    username = job_input["username"]
    repo_name = job_input["repo_name"]
    repo_url = f"https://{github_token}:x-oauth-basic@github.com/{username}/{repo_name}.git"

    # Docker image information
    dockerfile_path = f"./{repo_name}/Dockerfile"
    image_name = f"{username}/{repo_name}"
    image_tag = job_input.get("image_tag", "latest")

    # Clone the GitHub repository
    clone_repo(repo_url, repo_name)

    # Build the Docker image
    build_docker_image(dockerfile_path, image_name, image_tag)

    # Push the Docker image to GitHub Docker Registry
    result = push_docker_image(username, image_name, image_tag, github_token)
    
    return result
    
runpod.serverless.start({"handler": handler})

import runpod

import os
import git
import docker

def clone_repo(repo_url, repo_name):
    # Clone the GitHub repository
    git.Repo.clone_from(repo_url, repo_name)

def build_docker_image(dockerfile_path, image_name, image_tag):
    # Initialize Docker client
    client = docker.DockerClient(base_url='unix://var/run/docker.sock')

    # Build the Docker image
    image, build_logs = client.images.build(
        path=os.path.dirname(dockerfile_path),
        dockerfile=os.path.basename(dockerfile_path),
        tag=f"{image_name}:{image_tag}",
    )

    # Print build logs
    for log in build_logs:
        print(log)

def push_docker_image(username, image_name, image_tag, github_token):
    # Initialize Docker client
    client = docker.DockerClient(base_url='unix://var/run/docker.sock')

    # Authenticate with GitHub Docker Registry using access token
    client.login(username=username, password=github_token, registry="ghcr.io")

    # Push the Docker image to GitHub Docker Registry
    client.images.push(repository=f"ghcr.io/{image_name}", tag=image_tag)

def handler(job):
    job_input = job["input"]
    
    # GitHub access token
    github_token = os.environ["GITHUB_TOKEN"]

    # GitHub repository information
    username = job_input["username"]
    repo_name = job_input["repo_name"]
    repo_url = f"https://{github_token}:x-oauth-basic@github.com/{username}/{repo_name}.git"

    # Docker image information
    dockerfile_path = f"./{repo_name}/Dockerfile"
    image_name = f"{username}/{repo_name}"
    image_tag = job_input.get("image_tag", "latest")

    # Clone the GitHub repository
    clone_repo(repo_url, repo_name)

    # Build the Docker image
    build_docker_image(dockerfile_path, image_name, image_tag)

    # Push the Docker image to GitHub Docker Registry
    result = push_docker_image(username, image_name, image_tag, github_token)
    
    return result
    
runpod.serverless.start({"handler": handler})

M

Madiator2011•5/9/24, 2:11 PM

you cant run docker in docker

M

MBalkOP•5/9/24, 2:30 PM

is it not possible with something like this? https://stackoverflow.com/a/68975166

Stack Overflow

Python Docker SDK inside container

Im wondering if there is a way to use Python Docker SKD:
https://docker-py.readthedocs.io/en/stable/index.html#docker-sdk-for-python
inside a container and still be able to menage outside container...

M

Madiator2011•5/9/24, 2:32 PM

runpod pods are docker containers and it's not possible to run another docker in docker

J

J.•5/9/24, 3:21 PM

The objective is to run build a docker container and deploy it to runpod to run

M

MBalkOP•5/9/24, 3:24 PM

Yes, I want to clone a github repo, build the image, then push the image to ghcr. Because I cannot build certain images on my Macbook or Github Actions

MMBalk Yes, I want to clone a github repo, build the image, then push the image to ghcr...

J

J.•5/9/24, 3:32 PM

You can use CPU Pod, supposedly, I haven't tried

J

J.•5/9/24, 3:32 PM

If you are familiar with CLI

MMBalk Yes, I want to clone a github repo, build the image, then push the image to ghcr...

J

J.•5/9/24, 3:32 PM

Well also, i think u can specify --platform for the amd64 platform

J

J.•5/9/24, 3:32 PM

on mac, i believe

J

J.•5/9/24, 3:32 PM

with docker

J

J.•5/9/24, 3:33 PM

https://depot.dev/orgs

I use depot personally, but it costs money

Depot

Sign in | Depot

Sign in to Depot

J

J.•5/9/24, 3:33 PM

but they have a 7 day trial

Docker build inside serverless

Similar Threads

Similar Threads

Similar Threads