feat: V1.0 (#734)

Co-authored-by: Michael Neale <michael.neale@gmail.com> Co-authored-by: Wendy Tang <wendytang@squareup.com> Co-authored-by: Jarrod Sibbison <72240382+jsibbison-square@users.noreply.github.com> Co-authored-by: Alex Hancock <alex.hancock@example.com> Co-authored-by: Alex Hancock <alexhancock@block.xyz> Co-authored-by: Lifei Zhou <lifei@squareup.com> Co-authored-by: Wes <141185334+wesrblock@users.noreply.github.com> Co-authored-by: Max Novich <maksymstepanenko1990@gmail.com> Co-authored-by: Zaki Ali <zaki@squareup.com> Co-authored-by: Salman Mohammed <smohammed@squareup.com> Co-authored-by: Kalvin C <kalvinnchau@users.noreply.github.com> Co-authored-by: Alec Thomas <alec@swapoff.org> Co-authored-by: lily-de <119957291+lily-de@users.noreply.github.com> Co-authored-by: kalvinnchau <kalvin@block.xyz> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Rizel Scarlett <rizel@squareup.com> Co-authored-by: bwrage <bwrage@squareup.com> Co-authored-by: Kalvin Chau <kalvin@squareup.com> Co-authored-by: Alice Hau <110418948+ahau-square@users.noreply.github.com> Co-authored-by: Alistair Gray <ajgray@stripe.com> Co-authored-by: Nahiyan Khan <nahiyan.khan@gmail.com> Co-authored-by: Alex Hancock <alexhancock@squareup.com> Co-authored-by: Nahiyan Khan <nahiyan@squareup.com> Co-authored-by: marcelle <1852848+laanak08@users.noreply.github.com> Co-authored-by: Yingjie He <yingjiehe@block.xyz> Co-authored-by: Yingjie He <yingjiehe@squareup.com> Co-authored-by: Lily Delalande <ldelalande@block.xyz> Co-authored-by: Adewale Abati <acekyd01@gmail.com> Co-authored-by: Ebony Louis <ebony774@gmail.com> Co-authored-by: Angie Jones <jones.angie@gmail.com> Co-authored-by: Ebony Louis <55366651+EbonyLouis@users.noreply.github.com>
2025-01-24 13:04:43 -08:00
parent eccb1b2261
commit 1c9a7c0b05
688 changed files with 71147 additions and 19132 deletions
@@ -0,0 +1,11 @@
+# These variables are default initialization variables for locally hosted Langfuse server
+LANGFUSE_INIT_PROJECT_NAME=goose-local
+LANGFUSE_INIT_PROJECT_PUBLIC_KEY=publickey-local
+LANGFUSE_INIT_PROJECT_SECRET_KEY=secretkey-local
+LANGFUSE_INIT_USER_EMAIL=local@block.xyz
+LANGFUSE_INIT_USER_NAME=localdev
+LANGFUSE_INIT_USER_PASSWORD=localpwd
+
+LANGFUSE_INIT_ORG_ID=local-id
+LANGFUSE_INIT_ORG_NAME=local-org
+LANGFUSE_INIT_PROJECT_ID=goose
@@ -1,31 +1,131 @@
 services:
-  langfuse-server:
-    image: langfuse/langfuse:2
-    depends_on:
-      db:
+  langfuse-worker:
+    image: langfuse/langfuse-worker:3
+    restart: always
+    depends_on: &langfuse-depends-on
+      postgres:
        condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    ports:
+      - "3030:3030"
+    environment: &langfuse-worker-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      SALT: "mysalt"
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
+
+  langfuse-web:
+    image: langfuse/langfuse:3
+    restart: always
+    depends_on: *langfuse-depends-on
    ports:
      - "3000:3000"
    environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+      <<: *langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
+      NEXTAUTH_SECRET: mysecret
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: ${LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED:-false}
+      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
+      LANGFUSE_READ_FROM_CLICKHOUSE_ONLY: ${LANGFUSE_READ_FROM_CLICKHOUSE_ONLY:-true}
+      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}

-  db:
-    image: postgres
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    restart: always
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    restart: always
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  redis:
+    image: redis:7
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
    restart: always
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -33,14 +133,20 @@ services:
      timeout: 3s
      retries: 10
    environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
    ports:
      - 5432:5432
    volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data

 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
    driver: local
@@ -26,7 +26,7 @@ set -e
 SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
 LANGFUSE_DOCKER_COMPOSE_URL="https://raw.githubusercontent.com/langfuse/langfuse/main/docker-compose.yml"
 LANGFUSE_DOCKER_COMPOSE_FILE="langfuse-docker-compose.yaml"
-LANGFUSE_ENV_FILE="$SCRIPT_DIR/../packages/exchange/.env.langfuse.local"
+LANGFUSE_ENV_FILE="$SCRIPT_DIR/.env.langfuse.local"

 check_dependencies() {
    local dependencies=("curl" "docker")
@@ -80,6 +80,21 @@ launch_browser() {
    fi
 }

+get_project_keys() {
+    local public_key
+    local secret_key
+    
+    if [ -f "$LANGFUSE_ENV_FILE" ]; then
+        # Try to get keys from env file
+        public_key=$(grep -E "^LANGFUSE_INIT_PROJECT_PUBLIC_KEY=" "$LANGFUSE_ENV_FILE" | cut -d'=' -f2)
+        secret_key=$(grep -E "^LANGFUSE_INIT_PROJECT_SECRET_KEY=" "$LANGFUSE_ENV_FILE" | cut -d'=' -f2)
+    fi
+    
+    # Use environment variables if set, otherwise use defaults
+    echo "export LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${public_key:-$DEFAULT_PUBLIC_KEY}"
+    echo "export LANGFUSE_INIT_PROJECT_SECRET_KEY=${secret_key:-$DEFAULT_SECRET_KEY}"
+}
+
 print_login_variables() {
    if [ -f "$LANGFUSE_ENV_FILE" ]; then
        echo "If not already logged in use the following credentials to log in:"
@@ -87,6 +102,10 @@ print_login_variables() {
    else
        echo "Langfuse environment file with local credentials not found."
    fi
+    echo
+    echo "Project keys (add these to your environment):"
+    get_project_keys
+    echo
 }

 check_dependencies
@@ -96,4 +115,4 @@ start_docker_compose
 wait_for_service
 print_login_variables
 launch_browser
-popd > /dev/null
+popd > /dev/null