Edwin Salguero

feat: Integrate advanced analytics and enterprise UI

26a8ea5 over 1 year ago

10.4 kB

name: FRED ML CI/CD Pipeline

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main ]
  schedule:
    # Run tests daily at 2 AM UTC
    - cron: '0 2 * * *'

env:
  AWS_REGION: us-west-2
  S3_BUCKET: fredmlv1
  LAMBDA_FUNCTION: fred-ml-processor
  PYTHON_VERSION: '3.9'

jobs:
  # Test and Quality Checks
  test:
    name: 🧪 Test & Quality
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        
    - name: Cache pip dependencies
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install pytest pytest-cov black flake8 mypy
        
    - name: Run linting
      run: |
        echo "🔍 Running code linting..."
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics
        
    - name: Run type checking
      run: |
        echo "🔍 Running type checking..."
        mypy lambda/ frontend/ src/ --ignore-missing-imports
        
    - name: Run formatting check
      run: |
        echo "🎨 Checking code formatting..."
        black --check --diff .
        
    - name: Run unit tests
      run: |
        echo "🧪 Running unit tests..."
        pytest tests/unit/ -v --cov=lambda --cov=frontend --cov-report=xml
    
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      with:
        file: ./coverage.xml
        flags: unittests
        name: codecov-umbrella
        fail_ci_if_error: false

  # Integration Tests
  integration:
    name: 🔗 Integration Tests
    runs-on: ubuntu-latest
    needs: test
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ env.AWS_REGION }}
        
    - name: Run integration tests
      run: |
        echo "🔗 Running integration tests..."
        python scripts/test_complete_system.py --skip-e2e
      env:
        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
        S3_BUCKET: ${{ env.S3_BUCKET }}
        LAMBDA_FUNCTION: ${{ env.LAMBDA_FUNCTION }}

  # End-to-End Tests
  e2e:
    name: 🚀 End-to-End Tests
    runs-on: ubuntu-latest
    needs: [test, integration]
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ env.AWS_REGION }}
    
    - name: Run end-to-end tests
      run: |
        echo "🚀 Running end-to-end tests..."
        python scripts/test_complete_system.py
      env:
        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
        S3_BUCKET: ${{ env.S3_BUCKET }}
        LAMBDA_FUNCTION: ${{ env.LAMBDA_FUNCTION }}
        
    - name: Upload test results
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: test-results
        path: test_report.json

  # Security Scan
  security:
    name: 🔒 Security Scan
    runs-on: ubuntu-latest
    needs: test
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Run Bandit security scan
      run: |
        echo "🔒 Running security scan..."
        pip install bandit
        bandit -r lambda/ frontend/ src/ -f json -o bandit-report.json || true
        
    - name: Upload security report
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: security-report
        path: bandit-report.json

  # Build and Deploy Lambda
  deploy-lambda:
    name: ⚡ Deploy Lambda
    runs-on: ubuntu-latest
    needs: [test, integration, e2e, security]
    if: github.ref == 'refs/heads/main'
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.PYTHON_VERSION }}
        
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ env.AWS_REGION }}
        
    - name: Create Lambda deployment package
      run: |
        echo "📦 Creating Lambda deployment package..."
        cd lambda
        pip install -r requirements.txt -t .
        zip -r ../lambda-deployment.zip .
        cd ..
        
    - name: Update Lambda function
      run: |
        echo "⚡ Updating Lambda function..."
        aws lambda update-function-code \
          --function-name ${{ env.LAMBDA_FUNCTION }} \
          --zip-file fileb://lambda-deployment.zip \
          --region ${{ env.AWS_REGION }}
          
    - name: Update Lambda configuration
      run: |
        echo "⚙️  Updating Lambda configuration..."
        aws lambda update-function-configuration \
          --function-name ${{ env.LAMBDA_FUNCTION }} \
          --environment Variables="{S3_BUCKET=${{ env.S3_BUCKET }}}" \
          --region ${{ env.AWS_REGION }}
          
    - name: Update SSM parameter
      run: |
        echo "🔑 Updating FRED API key in SSM..."
        aws ssm put-parameter \
          --name "/fred-ml/api-key" \
          --value "${{ secrets.FRED_API_KEY }}" \
          --type "SecureString" \
          --overwrite \
          --region ${{ env.AWS_REGION }}
  # Deploy Infrastructure
  deploy-infrastructure:
    name: 🏗️ Deploy Infrastructure
    runs-on: ubuntu-latest
    needs: [test, integration, e2e, security]
    if: github.ref == 'refs/heads/main'
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ env.AWS_REGION }}
        
    - name: Deploy S3 bucket
      run: |
        echo "📦 Deploying S3 bucket..."
        aws cloudformation deploy \
          --template-file infrastructure/s3/bucket.yaml \
          --stack-name fredmlv1-s3-stack \
          --parameter-overrides BucketName=${{ env.S3_BUCKET }} \
          --capabilities CAPABILITY_NAMED_IAM \
          --region ${{ env.AWS_REGION }}
          
    - name: Deploy EventBridge rule
      run: |
        echo "⏰ Deploying EventBridge rule..."
        aws cloudformation deploy \
          --template-file infrastructure/eventbridge/quarterly-rule.yaml \
          --stack-name fred-ml-processor-eventbridge-stack \
          --parameter-overrides \
            LambdaFunctionName=${{ env.LAMBDA_FUNCTION }} \
            S3BucketName=${{ env.S3_BUCKET }} \
          --capabilities CAPABILITY_NAMED_IAM \
          --region ${{ env.AWS_REGION }}
  # Streamlit Cloud Deployment
  deploy-streamlit:
    name: 🎨 Deploy to Streamlit Cloud
    runs-on: ubuntu-latest
    needs: [deploy-lambda, deploy-infrastructure]
    if: github.ref == 'refs/heads/main'
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Deploy to Streamlit Cloud
      run: |
        echo "🎨 Deploying to Streamlit Cloud..."
        echo "Manual deployment required - follow instructions in docs/deployment/streamlit-cloud.md"
        echo "Repository is ready for Streamlit Cloud deployment"
        
    - name: Create deployment summary
      run: |
        echo "📋 Deployment Summary" > deployment-summary.md
        echo "===================" >> deployment-summary.md
        echo "" >> deployment-summary.md
        echo "✅ Lambda function updated" >> deployment-summary.md
        echo "✅ Infrastructure deployed" >> deployment-summary.md
        echo "📝 Streamlit Cloud deployment: Manual step required" >> deployment-summary.md
        echo "" >> deployment-summary.md
        echo "Next steps:" >> deployment-summary.md
        echo "1. Deploy to Streamlit Cloud using the web interface" >> deployment-summary.md
        echo "2. Configure environment variables in Streamlit Cloud" >> deployment-summary.md
        echo "3. Test the complete system" >> deployment-summary.md
        
    - name: Upload deployment summary
      uses: actions/upload-artifact@v3
      with:
        name: deployment-summary
        path: deployment-summary.md

  # Notifications
  notify:
    name: 📢 Notifications
    runs-on: ubuntu-latest
    needs: [deploy-streamlit]
    if: always()
    
    steps:
    - name: Download test results
      uses: actions/download-artifact@v3
      with:
        name: test-results
        
    - name: Download deployment summary
      uses: actions/download-artifact@v3
      with:
        name: deployment-summary
        
    - name: Send notification
      run: |
        echo "📢 Sending deployment notification..."
        if [ "${{ needs.deploy-streamlit.result }}" == "success" ]; then
          echo "✅ Deployment completed successfully!"
        else
          echo "❌ Deployment failed!"
        fi 

	name: FRED ML CI/CD Pipeline

	on:
	push:
	branches: [ main, develop ]
	pull_request:
	branches: [ main ]
	schedule:
	# Run tests daily at 2 AM UTC
	- cron: '0 2 * * *'

	env:
	AWS_REGION: us-west-2
	S3_BUCKET: fredmlv1
	LAMBDA_FUNCTION: fred-ml-processor
	PYTHON_VERSION: '3.9'

	jobs:
	# Test and Quality Checks
	test:
	name: 🧪 Test & Quality
	runs-on: ubuntu-latest

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python ${{ env.PYTHON_VERSION }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Cache pip dependencies
	uses: actions/cache@v3
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt
	pip install pytest pytest-cov black flake8 mypy

	- name: Run linting
	run: \|
	echo "🔍 Running code linting..."
	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics

	- name: Run type checking
	run: \|
	echo "🔍 Running type checking..."
	mypy lambda/ frontend/ src/ --ignore-missing-imports

	- name: Run formatting check
	run: \|
	echo "🎨 Checking code formatting..."
	black --check --diff .

	- name: Run unit tests
	run: \|
	echo "🧪 Running unit tests..."
	pytest tests/unit/ -v --cov=lambda --cov=frontend --cov-report=xml

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.xml
	flags: unittests
	name: codecov-umbrella
	fail_ci_if_error: false

	# Integration Tests
	integration:
	name: 🔗 Integration Tests
	runs-on: ubuntu-latest
	needs: test

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python ${{ env.PYTHON_VERSION }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
	aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	aws-region: ${{ env.AWS_REGION }}

	- name: Run integration tests
	run: \|
	echo "🔗 Running integration tests..."
	python scripts/test_complete_system.py --skip-e2e
	env:
	AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
	S3_BUCKET: ${{ env.S3_BUCKET }}
	LAMBDA_FUNCTION: ${{ env.LAMBDA_FUNCTION }}

	# End-to-End Tests
	e2e:
	name: 🚀 End-to-End Tests
	runs-on: ubuntu-latest
	needs: [test, integration]

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python ${{ env.PYTHON_VERSION }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
	aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	aws-region: ${{ env.AWS_REGION }}

	- name: Run end-to-end tests
	run: \|
	echo "🚀 Running end-to-end tests..."
	python scripts/test_complete_system.py
	env:
	AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
	S3_BUCKET: ${{ env.S3_BUCKET }}
	LAMBDA_FUNCTION: ${{ env.LAMBDA_FUNCTION }}

	- name: Upload test results
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: test-results
	path: test_report.json

	# Security Scan
	security:
	name: 🔒 Security Scan
	runs-on: ubuntu-latest
	needs: test

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Run Bandit security scan
	run: \|
	echo "🔒 Running security scan..."
	pip install bandit
	bandit -r lambda/ frontend/ src/ -f json -o bandit-report.json \|\| true

	- name: Upload security report
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: security-report
	path: bandit-report.json

	# Build and Deploy Lambda
	deploy-lambda:
	name: ⚡ Deploy Lambda
	runs-on: ubuntu-latest
	needs: [test, integration, e2e, security]
	if: github.ref == 'refs/heads/main'

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python ${{ env.PYTHON_VERSION }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
	aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	aws-region: ${{ env.AWS_REGION }}

	- name: Create Lambda deployment package
	run: \|
	echo "📦 Creating Lambda deployment package..."
	cd lambda
	pip install -r requirements.txt -t .
	zip -r ../lambda-deployment.zip .
	cd ..

	- name: Update Lambda function
	run: \|
	echo "⚡ Updating Lambda function..."
	aws lambda update-function-code \
	--function-name ${{ env.LAMBDA_FUNCTION }} \
	--zip-file fileb://lambda-deployment.zip \
	--region ${{ env.AWS_REGION }}

	- name: Update Lambda configuration
	run: \|
	echo "⚙️ Updating Lambda configuration..."
	aws lambda update-function-configuration \
	--function-name ${{ env.LAMBDA_FUNCTION }} \
	--environment Variables="{S3_BUCKET=${{ env.S3_BUCKET }}}" \
	--region ${{ env.AWS_REGION }}

	- name: Update SSM parameter
	run: \|
	echo "🔑 Updating FRED API key in SSM..."
	aws ssm put-parameter \
	--name "/fred-ml/api-key" \
	--value "${{ secrets.FRED_API_KEY }}" \
	--type "SecureString" \
	--overwrite \
	--region ${{ env.AWS_REGION }}

	# Deploy Infrastructure
	deploy-infrastructure:
	name: 🏗️ Deploy Infrastructure
	runs-on: ubuntu-latest
	needs: [test, integration, e2e, security]
	if: github.ref == 'refs/heads/main'

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
	aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	aws-region: ${{ env.AWS_REGION }}

	- name: Deploy S3 bucket
	run: \|
	echo "📦 Deploying S3 bucket..."
	aws cloudformation deploy \
	--template-file infrastructure/s3/bucket.yaml \
	--stack-name fredmlv1-s3-stack \
	--parameter-overrides BucketName=${{ env.S3_BUCKET }} \
	--capabilities CAPABILITY_NAMED_IAM \
	--region ${{ env.AWS_REGION }}

	- name: Deploy EventBridge rule
	run: \|
	echo "⏰ Deploying EventBridge rule..."
	aws cloudformation deploy \
	--template-file infrastructure/eventbridge/quarterly-rule.yaml \
	--stack-name fred-ml-processor-eventbridge-stack \
	--parameter-overrides \
	LambdaFunctionName=${{ env.LAMBDA_FUNCTION }} \
	S3BucketName=${{ env.S3_BUCKET }} \
	--capabilities CAPABILITY_NAMED_IAM \
	--region ${{ env.AWS_REGION }}

	# Streamlit Cloud Deployment
	deploy-streamlit:
	name: 🎨 Deploy to Streamlit Cloud
	runs-on: ubuntu-latest
	needs: [deploy-lambda, deploy-infrastructure]
	if: github.ref == 'refs/heads/main'

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Deploy to Streamlit Cloud
	run: \|
	echo "🎨 Deploying to Streamlit Cloud..."
	echo "Manual deployment required - follow instructions in docs/deployment/streamlit-cloud.md"
	echo "Repository is ready for Streamlit Cloud deployment"

	- name: Create deployment summary
	run: \|
	echo "📋 Deployment Summary" > deployment-summary.md
	echo "===================" >> deployment-summary.md
	echo "" >> deployment-summary.md
	echo "✅ Lambda function updated" >> deployment-summary.md
	echo "✅ Infrastructure deployed" >> deployment-summary.md
	echo "📝 Streamlit Cloud deployment: Manual step required" >> deployment-summary.md
	echo "" >> deployment-summary.md
	echo "Next steps:" >> deployment-summary.md
	echo "1. Deploy to Streamlit Cloud using the web interface" >> deployment-summary.md
	echo "2. Configure environment variables in Streamlit Cloud" >> deployment-summary.md
	echo "3. Test the complete system" >> deployment-summary.md

	- name: Upload deployment summary
	uses: actions/upload-artifact@v3
	with:
	name: deployment-summary
	path: deployment-summary.md

	# Notifications
	notify:
	name: 📢 Notifications
	runs-on: ubuntu-latest
	needs: [deploy-streamlit]
	if: always()

	steps:
	- name: Download test results
	uses: actions/download-artifact@v3
	with:
	name: test-results

	- name: Download deployment summary
	uses: actions/download-artifact@v3
	with:
	name: deployment-summary

	- name: Send notification
	run: \|
	echo "📢 Sending deployment notification..."
	if [ "${{ needs.deploy-streamlit.result }}" == "success" ]; then
	echo "✅ Deployment completed successfully!"
	else
	echo "❌ Deployment failed!"
	fi