chore(ob): add/simplify launcher cmds; add casdoor

This commit is contained in:
jerlendds
2023-10-28 07:26:14 -06:00
parent c6a17d46cd
commit aba6749c9d
20 changed files with 655 additions and 3 deletions

105
launcher Executable file
View File

@@ -0,0 +1,105 @@
#!/usr/bin/env sh
export OB_CMD=$1;
export OB_CMD_ARGS=$2;
export OB_DIR="$(dirname $0)/ob"
cd $OB_DIR
if [ -f cmd/env ]; then
source cmd/env;
fi
# localhost:8080/login/oauth/authorize?client_id=&response_type=&redirect_uri=&scope=read&state=
usage_containers () {
echo ""
echo -e "$(c 0uY) CONTAINERS: CONTAINER DESCRIPTION.$(c)"
echo -e " $(c Y )ui$(c): React - SPA frontend."
echo -e " $(c Y )backend$(c): FastAPI - running on Uvicorn with hot reloading."
echo -e " $(c Y )casdoor$(c): Casdoor - An open-source IAM / SSO platform with web UI supporting OAuth 2.0, OIDC, etc"
echo -e " $(c Y )worker$(c): Celery - Distributed Task Queue."
echo -e " $(c Y )clearly$(c): Clearly - Real time monitor for your celery tasks and workers."
echo -e " $(c Y )redis$(c): Redis - In-memory database that persists on disk. Used by worker and for caching."
echo -e " $(c Y )s3$(c): Minio S3 object storage - In the future we plan to also support seaweedfs."
echo -e " $(c Y )db$(c): PostgreSQL - need I say anything else?"
echo -e " $(c Y )sdb$(c): ScyllaDB - a NoSQL database that optimizes for raw performance. Used by JanusGraph."
# TODO: Make lucene the default option...
# > "Apache Lucene performs better in small scale, single machine applications..."
# source: https://docs.janusgraph.org/index-backend/
echo -e " $(c Y )index$(c): Solr - Open source enterprise search platform - Used by JanusGraph."
echo -e " $(c Y )janus$(c): JanusGraph - Distributed, open source, massively scalable graph database."
exit 1
}
usage () {
echo -e "Launcher usage: $(c G)./launcher COMMAND COMMAND_ARGS $(c)"
echo -e "$(c 0usBB) COMMANDS: COMMAND DESCRIPTION.$(c)"
echo -e " $(c YsB)start$(c): Start the OSINTBuddy app and tail the logs."
echo -e " $(c YsB)stop$(c): Stop the OSINTBuddy app (or a specific container ./launcher stop janus)."
echo -e " $(c YsB)build$(c): Build the OSINTBuddy app (or a specific container ./launcher build ui)."
echo -e " $(c YsB)restart$(c): Restart the app (or a specific container: ./launcher restart janus)."
echo -e " $(c YsB)api:gen$(c): Generate API client SDK from the backend OpenAPI spec; launch the UI outside of the container"
echo -e " $(c YsB)bootstrap$(c): Initialize and download dependencies and check for environment misconfigurations"
echo -e " $(c YsB)debug:send$(c): Upload all logs to https://paste.c-net.org for help with debugging"
echo -e " $(c YsB)cleanup$(c): Remove docker volumes."
echo -e " $(c YsB)shell$(c): Enter a containers shell."
echo -e " $(c YsB)migrate$(c): Generate a migration."
echo -e " $(c YsB)python$(c): Open a python shell on the backend."
echo -e " $(c YsB)ui$(c): Stop the UI container and launch the frontend locally."
echo -e " $(c YsB)containers$(c): List the names you can access the containers by"
usage_containers
exit 1
}
if [ -z "$OB_CMD" -a "$OB_CMD" != "cleanup" ]; then
usage
fi
case "$OB_CMD" in
bootstrap)
cmd/bootstrap
;;
start)
cmd/start
;;
restart)
cmd/restart
;;
stop)
cmd/stop
;;
build)
cmd/build
;;
logs)
cmd/logs
;;
api:gen)
cmd/api:gen
;;
ui)
cmd/api:gen
ob_compose stop ui && \
cd ../frontend && yarn start
;;
debug:send)
cmd/debug:send
;;
cleanup)
cmd/stop
cmd/cleanup
;;
shell)
cmd/shell
;;
python)
cmd/python
;;
migrate)
cmd/migrate
;;
containers)
usage_containers
;;
esac

30
ob/_casdoor/conf/app.conf Normal file
View File

@@ -0,0 +1,30 @@
appname = casdoor
httpport = 8080
runmode = dev
copyrequestbody = true
driverName = postgres
dataSourceName ="user=casdoor password=casdoorpassword host=db port=5432 sslmode=disable dbname=casdoor"
dbName = casdoor
tableNamePrefix =
showSql = false
redisEndpoint =
defaultStorageProvider =
isCloudIntranet = false
authState = "casdoor"
socks5Proxy = "127.0.0.1:10808"
verificationCodeTimeout = 10
initScore = 0
logPostOnly = true
origin =
originFrontend =
staticBaseUrl = "https://cdn.casbin.org"
isDemoMode = false
batchSize = 100
enableGzip = true
ldapServerPort = 389
radiusServerPort = 1812
radiusSecret = "secret"
quota = {"organization": -1, "user": -1, "application": -1, "provider": -1}
logConfig = {"filename": "logs/casdoor.log", "maxdays":99999, "perm":"0770"}
initDataFile = "./init_data.json"
frontendBaseDir = "../casdoor"

137
ob/alpha-compose.yml Executable file
View File

@@ -0,0 +1,137 @@
version: "3.9"
services:
ui:
tty: true
container_name: "ob_dev_ui"
build:
context: ../frontend
volumes:
- ../frontend/:/app/
- ../frontend/node_modules:/app/node_modules
ports:
- "${FRONTEND_PORT-3000}:3000"
env_file:
- .env.alpha
backend:
tty: true
container_name: "ob_dev_backend"
command: ["/bin/sh", "-c", "./start-reload.sh"]
build:
context: ../backend/backend
dockerfile: backend.Dockerfile
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:${BACKEND_PORT:-8000}/status"]
interval: 10s
timeout: 2s
retries: 5
volumes:
- ../backend/backend/app:/app/
depends_on:
- db
environment:
PYTHONDONTWRITEBYTECODE: 1
ports:
- "${BACKEND_PORT-8000}:80"
env_file:
- .env.alpha
casdoor:
container_name: "ob_dev_casdoor"
image: casbin/casdoor:latest
restart: on-failure
depends_on:
- backend
links:
- backend
ports:
- 8080:8080
volumes:
- ./_casdoor/conf:/conf
redis:
image: ${REDIS_IMAGE-redis:alpine3.18}
container_name: "ob_dev_redis"
ports:
- "${REDIS_PORT-6379}:6379"
command: redis-server
db:
image: "${POSTGRES_IMAGE-postgres:15.4}"
container_name: "ob_dev_db"
volumes:
- "ob-db-data:${PGDATA-/var/lib/postgresql/data/pgdata}"
env_file:
- .env.alpha
ports:
- "${POSTGRES_PORT-5432}:5432"
janus:
container_name: "ob_dev_janus"
build:
context: ./_janus
dockerfile: janusgraph.Dockerfile
ports:
- "8182:8182"
volumes:
- "./_janus/janusgraph-dynamic.properties:/opt/janusgraph/janusgraph-dynamic.properties"
- "./_janus/janusgraph-server.yaml:/etc/opt/janusgraph/janusgraph-server.yaml"
- "./_janus/empty-sample.groovy:/opt/janusgraph/scripts/empty-sample.groovy"
sdb:
image: ${SCYLLA_IMAGE-scylladb/scylla:5.1}
container_name: "ob_dev_sdb"
ports:
- "${SDB_REST_PORT-10000}:10000"
- "${CQL_PORT-9042}:9042"
- "${THRIFT_PORT-9160}:9160"
- "${INTERNODE_PORT-7000}:7000"
- "${INTERNODE_ONE_PORT-7001}:7001"
- "${JMX_PORT-7199}:7199"
env_file:
- .env.alpha
volumes:
- ob-sdb-data:${SCYLLADATA-/var/lib/scylla}
index:
image: "${SOLR_IMAGE-solr:9.3.0-slim}"
container_name: "ob_dev_index"
volumes:
- ob-solr-data:/var/solr
ports:
- "${SOLR_PORT-8983}:8983"
env_file:
- .env.alpha
# TODO: Fix worker and clearly
worker:
tty: true
container_name: "ob_dev_worker"
build:
context: ../backend/backend
dockerfile: worker.Dockerfile
command: ./worker-start.sh
env_file:
- .env.alpha
clearly:
tty: true
image: "${CLEARLY_IMAGE-rsalmei/clearly:latest}"
container_name: "ob_dev_clearly"
ports:
- "${CLEARLY_PORT-12223}:12223"
command:
- "server $(CELERY_BROKER_URL) --backend "
env_file:
- .env.alpha
s3: # TODO: Add option to use ob/_seaweedfs/* eventually
image: "${S3_IMAGE-minio/minio:RELEASE.2023-10-16T04-13-43Z}"
container_name: "ob_dev_s3"
ports:
- 9000:9000
- 9001:9001
command: 'server -master.port=9333 -volume.port=8080 -dir="/ob-data" --console-address ":9001"'
env_file:
- .env.alpha
# TODO: Move important stuff to secrets...
# secrets:
# POSTGRES_PASSWORD:
# file: ./???
volumes:
ob-db-data:
ob-sdb-data:
ob-solr-data:

17
ob/cmd/_colors Executable file
View File

@@ -0,0 +1,17 @@
#!/usr/bin/env bash
# https://github.com/ppo/bash-colors (v0.3.0)
c() { [ $# == 0 ] && printf "\e[0m" || printf "$1" | sed 's/\(.\)/\1;/g;s/\([SDIUFNHT]\)/2\1/g;s/\([KRGYBMCW]\)/3\1/g;s/\([krgybmcw]\)/4\1/g;y/SDIUFNHTsdiufnhtKRGYBMCWkrgybmcw/12345789123457890123456701234567/;s/^\(.*\);$/\\e[\1m/g'; }
cecho() { echo -e "$(c $1)${2}\e[0m"; }
# KISS Colors (v1): https://github.com/ppo/bash-colors/blob/master/kiss-colors.sh
if ((`tput colors` >= 8)); then
readonly reset="\e[0m" # Uppercase = bold.
readonly black="\e[0;30m"; readonly BLACK="\e[1;30m"
readonly red="\e[0;31m"; readonly RED="\e[1;31m"
readonly green="\e[0;32m"; readonly GREEN="\e[1;32m"
readonly yellow="\e[0;33m"; readonly YELLOW="\e[1;33m"
readonly blue="\e[0;34m"; readonly BLUE="\e[1;34m"
readonly magenta="\e[0;35m"; readonly MAGENTA="\e[1;35m"
readonly cyan="\e[0;36m"; readonly CYAN="\e[1;36m"
readonly white="\e[0;37m"; readonly WHITE="\e[1;37m"
fi

11
ob/cmd/api:gen Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env sh
. cmd/env
run_api_gen () {
heading "Generating client SDK..."
cd ../frontend && \
yarn swagger:codegen && \
success
}
run_api_gen

137
ob/cmd/bootstrap Executable file
View File

@@ -0,0 +1,137 @@
#!/usr/bin/env sh
. cmd/env
# From https://stackoverflow.com/a/44660519/702738
compare_version() {
if [[ $1 == $2 ]]; then
return 1
fi
local IFS=.
local i a=(${1%%[^0-9.]*}) b=(${2%%[^0-9.]*})
local arem=${1#${1%%[^0-9.]*}} brem=${2#${2%%[^0-9.]*}}
for ((i=0; i<${#a[@]} || i<${#b[@]}; i++)); do
if ((10#${a[i]:-0} < 10#${b[i]:-0})); then
return 1
elif ((10#${a[i]:-0} > 10#${b[i]:-0})); then
return 0
fi
done
if [ "$arem" '<' "$brem" ]; then
return 1
elif [ "$arem" '>' "$brem" ]; then
return 0
fi
return 1
}
check_images () {
# TODO: fix me
test_solr=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$SOLR_IMAGE"`
test_janus=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$JANUS_IMAGE"`
test_postgres=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$POSTGRES_IMAGE"`
test_s3=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$S3_IMAGE"`
test_clearly`$docker_path images | awk '{print $1 ":" $2 }' | grep "$CLEARLY_IMAGE"`
test_backend=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$BACKEND_IMAGE"`
test_nginx=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$NGINX_IMAGE"`
test_scylla=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$SCYLLA_IMAGE"`
test_node=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$NODE_IMAGE"`
test_redis=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$REDIS_IMAGE"`
# arm experimental support is on a fixed tag, always pull
if [ -z "$test_solr" ] || [ -z "$test_janus" ] || [ -z "$test_postgres" ] || [ -z "$test_s3" ] || [ -z "$test_clearly" ] || [ -z "$test_backend" ] || [ -z "$test_nginx" ] || [ -z "$test_scylla" ] || [ -z "$test_node" ] || [ -z "$test_redis" ] || [ $arm = true ]; then
echo
echo "WARNING: We are about to start downloading the base images for OSINTBuddy"
echo "This process may take anywhere between 10 minutes to an hour, depending on your network speed"
echo
echo "Please be patient"
echo
ob_compose pull
fi
}
install_docker() {
echo "Docker is not installed, you will need to install Docker in order to run the OSINTBuddy Launcher"
echo "See https://docs.docker.com/installation/"
exit 1
}
check_prereqs() {
if [ -z $docker_path ]; then
install_docker
fi
# 1. docker daemon running?
# we send stderr to /dev/null cause we don't care about warnings,
# it usually complains about swap which does not matter
test=`$docker_path info 2> /dev/null`
if [[ $? -ne 0 ]] ; then
echo "Cannot connect to the docker daemon - verify it is running and you have access"
exit 1
fi
# 2. running recommended docker version
test=($($docker_path --version)) # Get docker version string
test=${test[2]//,/} # Get version alone and strip comma if exists
# At least minimum docker version
if compare_version "${docker_min_version}" "${test}"; then
echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
exit 1
fi
# 3. docker images are downloadeded
# check_images
arm=false
case $(uname -m) in
armv7l)
error "ERROR: 32bit arm is not supported. Check if your hardware support arm64, which is supported in experimental capacity."
exit 1
;;
aarch64 | arm64)
warning "WARNING: Support for aarch64 is experimental at the moment. Please report any problems at https://forum.osintbuddy.com/c/bug-reports/6"
image=""
arm=true
;;
x86_64)
success "x86_64 arch detected."
;;
*)
error "ERROR: unknown arch detected."
exit 1
;;
esac
# 4. running recommended git version
test=($($git_path --version)) # Get git version string
test=${test[2]//,/} # Get version alone and strip comma if exists
# At least minimum version
if compare_version "${git_min_version}" "${test}"; then
warning "ERROR: Git version ${test} not supported, please upgrade to at least ${git_min_version}, or recommended ${git_rec_version}"
exit 1
fi
# Recommend best version
if compare_version "${git_rec_version}" "${test}"; then
warning "WARNING: Git version ${test} deprecated, recommend upgrade to ${git_rec_version} or newer."
fi
}
run_bootstrap() {
trap 'abort' INT
heading "Checking for prerequisities..."
check_prereqs
heading "Building containers..."
ob_compose build
success
info "You can now launch OSINTBuddy with \`./launcher start\`"
}
run_bootstrap

14
ob/cmd/build Executable file
View File

@@ -0,0 +1,14 @@
#!/usr/bin/env sh
. cmd/env
run_build() {
if [ ! -z "$OB_CMD" ] & [ ! -z "$OB_CMD_ARGS" ]; then
ob_compose build $OB_CMD_ARGS
exit 0
else
ob_compose build
exit 0
fi
}
run_build

13
ob/cmd/cleanup Executable file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/env sh
. cmd/env
run_cleanup() {
heading "Stopping containers and removing volumes"
ob_compose down
$docker_path volume rm $($docker_path volume ls -q) # &> /dev/null
success "Finished cleanup."
exit 0
}
run_cleanup

View File

@@ -1,2 +1,2 @@
#!/usr/bin/env bash
docker ps -aqf "name=$(yq -r .services.$1.container_name docker-compose.yml)"
#!/usr/bin/env sh
docker ps -aqf "name=$(yq -r .services.$1.container_name alpha-compose.yml)"

15
ob/cmd/container_ids Executable file
View File

@@ -0,0 +1,15 @@
#!/usr/bin/env sh
. cmd/env
export_container_ids() {
export ID_UI=$(cmd/container_id "ui");
export ID_BACKEND=$(cmd/container_id "backend");
export ID_REDIS=$(cmd/container_id "redis");
export ID_DB=$(cmd/container_id "db");
export ID_JANUS=$(cmd/container_id "janus");
export ID_SDB=$(cmd/container_id "sdb");
export ID_INDEX=$(cmd/container_id "index");
export ID_WORKER=$(cmd/container_id "worker");
export ID_CLEARLY=$(cmd/container_id "clearly");
export ID_S3=$(cmd/container_id "s3");
}

27
ob/cmd/debug:send Executable file
View File

@@ -0,0 +1,27 @@
#!/usr/bin/env sh
. cmd/env
pastebin() {
ob_compose logs > osintbuddy-debug.log
sed 's/\x1b\[[0-9;]*[a-zA-Z]//g' osintbuddy-debug.log > debug.log
rm osintbuddy-debug.log
local url='https://paste.c-net.org/'
local file='debug.log'
curl -s \
--data-binary @"$file" \
--header "X-FileName: ${file##*/}" \
"$url"
# curl -s --data-binary @- "$url"
rm debug.log
}
run_pastebin () {
trap 'abort' INT
heading "Uploading container logs to: https://paste.c-net.org/"
pastebin
success
exit 0
}
run_pastebin

17
ob/cmd/env Executable file
View File

@@ -0,0 +1,17 @@
#!/usr/bin/env sh
source cmd/_colors
config_file="alpha-compose.yml"
docker_path=`which docker`
git_path=`which git`
alias ob_compose="$docker_path compose -f $config_file"
# Colors...
function heading() { printf "$(c B1)[OSINTBuddy: $OB_CMD] ${WHITE}${1}${reset}\n"; }
function warning() { printf "${BLUE}[OSINTBuddy: $OB_CMD] ${magenta}${1}${reset}\n"; }
function error() { printf "${BLUE}[OSINTBuddy: $OB_CMD] ${red}${1}${reset}\n"; }
function success() { printf "${BLUE}[OSINTBuddy: $OB_CMD] ${green}${1:-Completed successfully!}${reset}\n"; }
function info() { printf "${BLUE}[OSINTBuddy: $OB_CMD] ${CYAN}${1:-''}${reset}\n"; }
function abort() { printf "${BLUE}[OSINTBuddy: $OB_CMD] ${RED}${1:-Aborted by user!}${reset}\n\n"; exit; }

13
ob/cmd/logs Executable file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/env sh
. cmd/env
run_logs() {
heading "Loading $OB_CMD_ARGS logs"
if [ ! -z "$OB_CMD_ARGS" ]; then
ob_compose logs $OB_CMD_ARGS -f
else
ob_compose logs -f
fi
}
run_logs

25
ob/cmd/migrate Executable file
View File

@@ -0,0 +1,25 @@
#!/usr/bin/env sh
. cmd/env
migration_message="$OB_CMD_ARGS"
run_migration() {
trap 'abort' INT
heading "Preparing to create a new migration..."
if [ ! -z "$migration_message" ]; then
ob_compose exec backend alembic revision --autogenerate -m "$migration_message"
ob_compose exec backend alembic upgrade head
heading "Fixing migration file permissions..."
ob_compose exec backend sh -c "/usr/sbin/useradd $USER > /dev/null 2>&1"
ob_compose exec backend sh -c "chown $USER:$USER -R alembic/versions/*.py"
success
info "View migrations at: ./backend/backend/app/alembic/versions"
exit 0
else
error "Error: A short message summarizing your changes is required to create an alembic migration:"
info " e.g: ./launcher migrate 'update entities is_favorite'"
fi
}
run_migration $migration_message

19
ob/cmd/python Executable file
View File

@@ -0,0 +1,19 @@
#!/usr/bin/env sh
. cmd/env
run_python() {
trap 'abort' INT
heading "Launching backend Python shell..."
BACKEND_ID=$(cmd/container_id backend)
if [ ! -z "$BACKEND_ID" ]; then
ob_compose exec backend python3 -i app/shell.py
success
else
error "Error: The backend container is not running."
info " To start up the OSINTBuddy app try running: ./launcher start"
fi
}
run_python

15
ob/cmd/restart Executable file
View File

@@ -0,0 +1,15 @@
#!/usr/bin/env sh
. cmd/env
run_restart() {
if [ ! -z "$OB_CMD" ] & [ ! -z "$OB_CMD_ARGS" ]; then
ob_compose restart $OB_CMD_ARGS;
ob_compose logs $OB_CMD_ARGS -f
exit 0
else
ob_compose restart;
exit 0
fi
}
run_restart

View File

@@ -1 +0,0 @@
#!/usr/bin/env bash

11
ob/cmd/shell Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env sh
. cmd/env
run_shell() {
echo "Launching the $OB_CMD_ARGS shell"
ob_compose exec $OB_CMD_ARGS sh
exit 0
}
run_shell

32
ob/cmd/start Executable file
View File

@@ -0,0 +1,32 @@
#!/usr/bin/env sh
. cmd/env
start() {
ob_compose up -d
}
run_start() {
unset ERR
start || ERR=$?
if [[ $ERR > 0 ]]; then
error "We ran into an error starting OSINTBuddy. Please file an issue on Github."
exit 1
else
ob_compose logs -f
success "Starting OSINTBuddy..."
success "After JanusGraph connects to ScyllaDB and Solr, the app is ready for use."
success "Please allow a few minutes for JanusGraph to finish starting up."
echo ""
info "To monitor when JanusGraph is ready, run: ./launcher logs janus"
info "After that you can access OSINTBuddy through the following URLs:"
info " http://localhost:3000 - frontend"
info " http://localhost:8080 - casdoor"
info " http://localhost:8000/api - backend"
info " http://localhost:8000/docs - docs"
exit 0
fi
}
run_start

15
ob/cmd/stop Executable file
View File

@@ -0,0 +1,15 @@
#!/usr/bin/env sh
. cmd/env
run_stop() {
if [ ! -z "$OB_CMD" ] & [ ! -z "$OB_CMD_ARGS" ]; then
ob_compose stop $OB_CMD_ARGS
exit 0
else
ob_compose stop
exit 0
fi
}
run_stop