mirror of https://github.com/CIRCL/AIL-framework
Merge remote-tracking branch 'origin/master' into dev
terrtia
commit
b1ec6bd6bf
|
|
@ -138,7 +138,7 @@ CIRCL organises training on how to use or extend the AIL framework. AIL training
|
|||
|
||||
## API
|
||||
|
||||
The API documentation is available in [doc/README.md](doc/README.md)
|
||||
The API documentation is available in [doc/api.md](doc/api.md)
|
||||
|
||||
## HOWTO
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
build/conf/sign.json
|
||||
build/conf/tracker.json
|
||||
images/
|
||||
|
||||
|
|
@ -0,0 +1,488 @@
|
|||
#!/bin/bash
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
setVars() {
|
||||
STORAGE_POOL_NAME=$(generateName "AIL")
|
||||
NETWORK_NAME=$(generateName "AIL")
|
||||
NETWORK_NAME=${NETWORK_NAME:0:14}
|
||||
PROFILE=$(generateName "AIL")
|
||||
|
||||
UBUNTU="ubuntu:22.04"
|
||||
}
|
||||
|
||||
setDefaults(){
|
||||
default_ail_project=$(generateName "AIL")
|
||||
default_ail_name=$(generateName "AIL")
|
||||
default_lacus="Yes"
|
||||
default_lacus_name=$(generateName "LACUS")
|
||||
default_partition=""
|
||||
}
|
||||
|
||||
error() {
|
||||
echo -e "${RED}ERROR: $1${NC}"
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo -e "${YELLOW}WARNING: $1${NC}"
|
||||
}
|
||||
|
||||
info() {
|
||||
echo -e "${BLUE}INFO: $1${NC}"
|
||||
}
|
||||
|
||||
success() {
|
||||
echo -e "${GREEN}SUCCESS: $1${NC}"
|
||||
}
|
||||
|
||||
err() {
|
||||
local parent_lineno="$1"
|
||||
local message="$2"
|
||||
local code="${3:-1}"
|
||||
|
||||
if [[ -n "$message" ]] ; then
|
||||
error "Line ${parent_lineno}: ${message}: exiting with status ${code}"
|
||||
else
|
||||
error "Line ${parent_lineno}: exiting with status ${code}"
|
||||
fi
|
||||
|
||||
deleteLXDProject "$PROJECT_NAME"
|
||||
lxc storage delete "$APP_STORAGE"
|
||||
lxc storage delete "$DB_STORAGE"
|
||||
lxc network delete "$NETWORK_NAME"
|
||||
exit "${code}"
|
||||
}
|
||||
|
||||
generateName(){
|
||||
local name="$1"
|
||||
echo "${name}-$(date +%Y%m%d%H%M%S)"
|
||||
}
|
||||
|
||||
setupLXD(){
|
||||
lxc project create "$PROJECT_NAME"
|
||||
lxc project switch "$PROJECT_NAME"
|
||||
|
||||
if checkRessourceExist "storage" "$STORAGE_POOL_NAME"; then
|
||||
error "Storage '$STORAGE_POOL_NAME' already exists."
|
||||
exit 1
|
||||
fi
|
||||
lxc storage create "$STORAGE_POOL_NAME" zfs source="$PARTITION"
|
||||
|
||||
if checkRessourceExist "network" "$NETWORK_NAME"; then
|
||||
error "Network '$NETWORK_NAME' already exists."
|
||||
fi
|
||||
lxc network create "$NETWORK_NAME" --type=bridge
|
||||
|
||||
if checkRessourceExist "profile" "$PROFILE"; then
|
||||
error "Profile '$PROFILE' already exists."
|
||||
fi
|
||||
lxc profile create "$PROFILE"
|
||||
lxc profile device add "$PROFILE" root disk path="/" pool="$STORAGE_POOL_NAME"
|
||||
lxc profile device add "$PROFILE" eth0 nic name=eth0 network="$NETWORK_NAME"
|
||||
}
|
||||
|
||||
waitForContainer() {
|
||||
local container_name="$1"
|
||||
|
||||
sleep 3
|
||||
while true; do
|
||||
status=$(lxc list --format=json | jq -e --arg name "$container_name" '.[] | select(.name == $name) | .status')
|
||||
if [ "$status" = "\"Running\"" ]; then
|
||||
echo -e "${BLUE}$container_name ${GREEN}is running.${NC}"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for $container_name container to start."
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
interrupt() {
|
||||
warn "Script interrupted by user. Delete project and exit ..."
|
||||
deleteLXDProject "$PROJECT_NAME"
|
||||
lxc network delete "$NETWORK_NAME"
|
||||
exit 130
|
||||
}
|
||||
|
||||
deleteLXDProject(){
|
||||
local project="$1"
|
||||
|
||||
echo "Starting cleanup ..."
|
||||
echo "Deleting container in project"
|
||||
for container in $(lxc query "/1.0/containers?recursion=1&project=${project}" | jq .[].name -r); do
|
||||
lxc delete --project "${project}" -f "${container}"
|
||||
done
|
||||
|
||||
echo "Deleting images in project"
|
||||
for image in $(lxc query "/1.0/images?recursion=1&project=${project}" | jq .[].fingerprint -r); do
|
||||
lxc image delete --project "${project}" "${image}"
|
||||
done
|
||||
|
||||
echo "Deleting profiles in project"
|
||||
for profile in $(lxc query "/1.0/profiles?recursion=1&project=${project}" | jq .[].name -r); do
|
||||
if [ "${profile}" = "default" ]; then
|
||||
printf 'config: {}\ndevices: {}' | lxc profile edit --project "${project}" default
|
||||
continue
|
||||
fi
|
||||
lxc profile delete --project "${project}" "${profile}"
|
||||
done
|
||||
|
||||
echo "Deleting project"
|
||||
lxc project delete "${project}"
|
||||
}
|
||||
|
||||
createAILContainer(){
|
||||
lxc launch $UBUNTU "$AIL_CONTAINER" --profile "$PROFILE"
|
||||
waitForContainer "$AIL_CONTAINER"
|
||||
lxc exec "$AIL_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf
|
||||
lxc exec "$AIL_CONTAINER" -- apt update
|
||||
lxc exec "$AIL_CONTAINER" -- apt upgrade -y
|
||||
lxc exec "$AIL_CONTAINER" -- useradd -m -s /bin/bash ail
|
||||
if lxc exec "$AIL_CONTAINER" -- id ail; then
|
||||
lxc exec "$AIL_CONTAINER" -- usermod -aG sudo ail
|
||||
success "User ail created."
|
||||
else
|
||||
error "User ail not created."
|
||||
exit 1
|
||||
fi
|
||||
lxc exec "$AIL_CONTAINER" -- bash -c "echo 'ail ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/ail"
|
||||
lxc exec "$AIL_CONTAINER" --cwd=/home/ail -- sudo -u ail bash -c "git clone https://github.com/ail-project/ail-framework.git"
|
||||
lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework -- sudo -u ail bash -c "./installing_deps.sh"
|
||||
lxc exec "$AIL_CONTAINER" -- sed -i '/^\[Flask\]/,/^\[/ s/host = 127\.0\.0\.1/host = 0.0.0.0/' /home/ail/ail-framework/configs/core.cfg
|
||||
lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework/bin -- sudo -u ail bash -c "./LAUNCH.sh -l"
|
||||
lxc exec "$AIL_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf
|
||||
}
|
||||
|
||||
createLacusContainer(){
|
||||
lxc launch $UBUNTU "$LACUS_CONTAINER" --profile "$PROFILE"
|
||||
waitForContainer "$LACUS_CONTAINER"
|
||||
lxc exec "$LACUS_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf
|
||||
lxc exec "$LACUS_CONTAINER" -- apt update
|
||||
lxc exec "$LACUS_CONTAINER" -- apt upgrade -y
|
||||
lxc exec "$LACUS_CONTAINER" -- apt install pipx -y
|
||||
lxc exec "$LACUS_CONTAINER" -- pipx install poetry
|
||||
lxc exec "$LACUS_CONTAINER" -- pipx ensurepath
|
||||
lxc exec "$LACUS_CONTAINER" -- apt install build-essential tcl -y
|
||||
lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/redis/redis.git
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- git checkout 7.2
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make test
|
||||
lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install
|
||||
AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p")
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps"
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env"
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && echo 'no' | /root/.local/bin/poetry run update --init"
|
||||
# Install Tor
|
||||
lxc exec "$LACUS_CONTAINER" -- apt install apt-transport-https -y
|
||||
lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list"
|
||||
lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list"
|
||||
lxc exec "$LACUS_CONTAINER" -- bash -c "wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg > /dev/null"
|
||||
lxc exec "$LACUS_CONTAINER" -- apt update
|
||||
lxc exec "$LACUS_CONTAINER" -- apt install tor deb.torproject.org-keyring -y
|
||||
lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf
|
||||
# Start Lacus
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./config/logging.json.sample ./config/logging.json
|
||||
lxc file push ./systemd/lacus.service "$LACUS_CONTAINER"/etc/systemd/system/lacus.service
|
||||
lxc exec "$LACUS_CONTAINER" -- systemctl daemon-reload
|
||||
lxc exec "$LACUS_CONTAINER" -- systemctl enable lacus.service
|
||||
lxc exec "$LACUS_CONTAINER" -- systemctl start lacus.service
|
||||
}
|
||||
|
||||
interactiveConfig(){
|
||||
echo
|
||||
echo "################################################################################"
|
||||
echo -e "# Welcome to the ${BLUE}AIL-framework-LXD${NC} Installer Script #"
|
||||
echo "#------------------------------------------------------------------------------#"
|
||||
echo -e "# This installer script will guide you through the installation process of #"
|
||||
echo -e "# ${BLUE}AIL${NC} using LXD. #"
|
||||
echo -e "# #"
|
||||
echo "################################################################################"
|
||||
echo
|
||||
|
||||
declare -A nameCheckArray
|
||||
|
||||
# Ask for LXD project name
|
||||
while true; do
|
||||
read -r -p "Name of the AIL LXD-project (default: $default_ail_project): " ail_project
|
||||
PROJECT_NAME=${ail_project:-$default_ail_project}
|
||||
if ! checkNamingConvention "$PROJECT_NAME"; then
|
||||
continue
|
||||
fi
|
||||
if checkRessourceExist "project" "$PROJECT_NAME"; then
|
||||
error "Project '$PROJECT_NAME' already exists."
|
||||
continue
|
||||
fi
|
||||
break
|
||||
done
|
||||
|
||||
# Ask for AIL container name
|
||||
while true; do
|
||||
read -r -p "Name of the AIL container (default: $default_ail_name): " ail_name
|
||||
AIL_CONTAINER=${ail_name:-$default_ail_name}
|
||||
if [[ ${nameCheckArray[$AIL_CONTAINER]+_} ]]; then
|
||||
error "Name '$AIL_CONTAINER' has already been used. Please choose a different name."
|
||||
continue
|
||||
fi
|
||||
if ! checkNamingConvention "$AIL_CONTAINER"; then
|
||||
continue
|
||||
fi
|
||||
nameCheckArray[$AIL_CONTAINER]=1
|
||||
break
|
||||
done
|
||||
|
||||
# Ask for Lacus installation
|
||||
read -r -p "Do you want to install Lacus (y/n, default: $default_lacus): " lacus
|
||||
lacus=${lacus:-$default_lacus}
|
||||
LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false)
|
||||
if $LACUS; then
|
||||
# Ask for LACUS container name
|
||||
while true; do
|
||||
read -r -p "Name of the Lacus container (default: $default_lacus_name): " lacus_name
|
||||
LACUS_CONTAINER=${lacus_name:-$default_lacus_name}
|
||||
if [[ ${nameCheckArray[$LACUS_CONTAINER]+_} ]]; then
|
||||
error "Name '$LACUS_CONTAINER' has already been used. Please choose a different name."
|
||||
continue
|
||||
fi
|
||||
if ! checkNamingConvention "$LACUS_CONTAINER"; then
|
||||
continue
|
||||
fi
|
||||
nameCheckArray[$LACUS_CONTAINER]=1
|
||||
break
|
||||
done
|
||||
|
||||
fi
|
||||
|
||||
# Ask for dedicated partitions
|
||||
read -r -p "Dedicated partition for AIL LXD-project (leave blank if none): " partition
|
||||
PARTITION=${partition:-$default_partition}
|
||||
|
||||
# Output values set by the user
|
||||
echo -e "\nValues set:"
|
||||
echo "--------------------------------------------------------------------------------------------------------------------"
|
||||
echo -e "PROJECT_NAME: ${GREEN}$PROJECT_NAME${NC}"
|
||||
echo "--------------------------------------------------------------------------------------------------------------------"
|
||||
echo -e "AIL_CONTAINER: ${GREEN}$AIL_CONTAINER${NC}"
|
||||
echo "--------------------------------------------------------------------------------------------------------------------"
|
||||
echo -e "LACUS: ${GREEN}$LACUS${NC}"
|
||||
if $LACUS; then
|
||||
echo -e "LACUS_CONTAINER: ${GREEN}$LACUS_CONTAINER${NC}"
|
||||
echo "--------------------------------------------------------------------------------------------------------------------"
|
||||
fi
|
||||
echo -e "PARTITION: ${GREEN}$PARTITION${NC}"
|
||||
echo "--------------------------------------------------------------------------------------------------------------------"
|
||||
|
||||
# Ask for confirmation
|
||||
read -r -p "Do you want to proceed with the installation? (y/n): " confirm
|
||||
confirm=${confirm:-$default_confirm}
|
||||
if [[ $confirm != "y" ]]; then
|
||||
warn "Installation aborted."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
nonInteractiveConfig(){
|
||||
VALID_ARGS=$(getopt -o h --long help,production,project:ail-name:,no-lacus,lacus-name:,partition: -- "$@")
|
||||
if [[ $? -ne 0 ]]; then
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
eval set -- "$VALID_ARGS"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-h | --help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--partition)
|
||||
partition=$2
|
||||
shift 2
|
||||
;;
|
||||
--project)
|
||||
ail_project=$2
|
||||
shift 2
|
||||
;;
|
||||
--ail-name)
|
||||
ail_name=$2
|
||||
shift 2
|
||||
;;
|
||||
--no-lacus)
|
||||
lacus="N"
|
||||
shift
|
||||
;;
|
||||
--lacus-name)
|
||||
lacus_name=$2
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Set global values
|
||||
PROJECT_NAME=${ail_project:-$default_ail_project}
|
||||
AIL_CONTAINER=${ail_name:-$default_ail_name}
|
||||
lacus=${lacus:-$default_lacus}
|
||||
LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false)
|
||||
LACUS_CONTAINER=${lacus_name:-$default_lacus_name}
|
||||
PARTITION=${partition:-$default_partition}
|
||||
}
|
||||
|
||||
validateArgs(){
|
||||
# Check Names
|
||||
local names=("$PROJECT_NAME" "$AIL_CONTAINER")
|
||||
for i in "${names[@]}"; do
|
||||
if ! checkNamingConvention "$i"; then
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if $LACUS && ! checkNamingConvention "$LACUS_CONTAINER"; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for Project
|
||||
if checkRessourceExist "project" "$PROJECT_NAME"; then
|
||||
error "Project '$PROJECT_NAME' already exists."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check Container Names
|
||||
local containers=("$AIL_CONTAINER")
|
||||
|
||||
declare -A name_counts
|
||||
for name in "${containers[@]}"; do
|
||||
((name_counts["$name"]++))
|
||||
done
|
||||
|
||||
if $LACUS;then
|
||||
((name_counts["$LACUS_CONTAINER"]++))
|
||||
fi
|
||||
|
||||
for name in "${!name_counts[@]}"; do
|
||||
if ((name_counts["$name"] >= 2)); then
|
||||
error "At least two container have the same name: $name"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
checkRessourceExist() {
|
||||
local resource_type="$1"
|
||||
local resource_name="$2"
|
||||
|
||||
case "$resource_type" in
|
||||
"container")
|
||||
lxc info "$resource_name" &>/dev/null
|
||||
;;
|
||||
"image")
|
||||
lxc image list --format=json | jq -e --arg alias "$resource_name" '.[] | select(.aliases[].name == $alias) | .fingerprint' &>/dev/null
|
||||
;;
|
||||
"project")
|
||||
lxc project list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null
|
||||
;;
|
||||
"storage")
|
||||
lxc storage list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null
|
||||
;;
|
||||
"network")
|
||||
lxc network list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null
|
||||
;;
|
||||
"profile")
|
||||
lxc profile list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null
|
||||
;;
|
||||
esac
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
checkNamingConvention(){
|
||||
local input="$1"
|
||||
local pattern="^[a-zA-Z0-9-]+$"
|
||||
|
||||
if ! [[ "$input" =~ $pattern ]]; then
|
||||
error "Invalid Name $input. Please use only alphanumeric characters and hyphens."
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -h, --help Display this help message and exit."
|
||||
echo " --project <project_name> Specify the project name."
|
||||
echo " --ail-name <container_name> Specify the AIL container name."
|
||||
echo " --no-lacus Do not create Lacus container."
|
||||
echo " --lacus-name <container_name> Specify the Lacus container name."
|
||||
echo " -i, --interactive Run the script in interactive mode."
|
||||
echo
|
||||
echo "This script sets up LXD containers for AIL and optionally Lacus."
|
||||
echo "It creates a new LXD project, and configures the network and storage."
|
||||
echo "Then it launches and configures the specified containers."
|
||||
echo
|
||||
echo "Examples:"
|
||||
echo " $0 --project myProject --ail-name ailContainer"
|
||||
echo " $0 --interactive"
|
||||
}
|
||||
|
||||
# ------------------ MAIN ------------------
|
||||
|
||||
setDefaults
|
||||
|
||||
# Check if interactive mode
|
||||
INTERACTIVE=false
|
||||
for arg in "$@"; do
|
||||
if [[ $arg == "-i" ]] || [[ $arg == "--interactive" ]]; then
|
||||
INTERACTIVE=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$INTERACTIVE" = true ]; then
|
||||
interactiveConfig
|
||||
else
|
||||
nonInteractiveConfig "$@"
|
||||
fi
|
||||
|
||||
validateArgs
|
||||
setVars
|
||||
|
||||
trap 'interrupt' INT
|
||||
trap 'err ${LINENO}' ERR
|
||||
|
||||
info "Setup LXD Project"
|
||||
setupLXD
|
||||
|
||||
info "Create AIL Container"
|
||||
createAILContainer
|
||||
|
||||
if $LACUS; then
|
||||
info "Create Lacus Container"
|
||||
createLacusContainer
|
||||
fi
|
||||
|
||||
# Print info
|
||||
ail_ip=$(lxc list "$AIL_CONTAINER" --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address')
|
||||
ail_email=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^email=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2")
|
||||
ail_password=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2")
|
||||
ail_API_Key=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^API_Key=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2")
|
||||
if $LACUS; then
|
||||
lacus_ip=$(lxc list "$LACUS_CONTAINER" --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address')
|
||||
fi
|
||||
echo "--------------------------------------------------------------------------------------------"
|
||||
echo -e "${BLUE}AIL ${NC}is up and running on $ail_ip"
|
||||
echo "--------------------------------------------------------------------------------------------"
|
||||
echo -e "${BLUE}AIL ${NC}credentials:"
|
||||
echo -e "Email: ${GREEN}$ail_email${NC}"
|
||||
echo -e "Password: ${GREEN}$ail_password${NC}"
|
||||
echo -e "API Key: ${GREEN}$ail_API_Key${NC}"
|
||||
echo "--------------------------------------------------------------------------------------------"
|
||||
if $LACUS; then
|
||||
echo -e "${BLUE}Lacus ${NC}is up and running on $lacus_ip"
|
||||
fi
|
||||
echo "--------------------------------------------------------------------------------------------"
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
# AIL-framework-LXD
|
||||
This installer is based on the [LXD](https://canonical.com/lxd) container manager and can be used to install AIL on Linux. It also supports the installation of [Lacus](https://github.com/ail-project/lacus) a crawler for the AIL framework.
|
||||
|
||||
## Requirements
|
||||
- [LXD](https://canonical.com/lxd) 5.19
|
||||
- jq 1.6
|
||||
|
||||
## Usage
|
||||
Make sure you have all the requirements installed on your system.
|
||||
|
||||
### Interactive mode
|
||||
Run the INSTALL.sh script with the --interactive flag to enter the interactive mode, which guides you through the configuration process:
|
||||
```bash
|
||||
bash INSTALL.sh --interactive
|
||||
```
|
||||
|
||||
### Non-interactive mode
|
||||
If you want to install AIL without the interactive mode, you can use the following command:
|
||||
```bash
|
||||
bash INSTALL.sh [OPTIONS]
|
||||
```
|
||||
|
||||
The following options are available:
|
||||
| Flag | Default Value | Description |
|
||||
| ------------------------------- | ----------------------- | ------------------------------------------------------------------------ |
|
||||
| `-i`, `--interactive` | N/A | Activates an interactive installation process. |
|
||||
| `--project <project_name>` | `AIL-<creation_time>` | Name of the LXD project for organizing and running the containers. |
|
||||
| `--ail-name <container_name>` | `AIL-<creation_time>` | The name of the container responsible for running the AIL application. |
|
||||
| `--no-lacus` | `false` | Determines whether to install the Lacus container. |
|
||||
| `--lacus-name <container_name>` | `LACUS-<creation_time>` | The name of the container responsible for running the Lacus application. |
|
||||
| `--partition <partition>` | `<none>` | Dedicated partition for LXD-project storage. |
|
||||
|
||||
|
||||
## Configuration
|
||||
If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO.md](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md)
|
||||
|
||||
## Using Images to run AIL
|
||||
If you want to use images to install AIL, you can download them from the ail-project [image website](https://images.ail-project.org/)
|
||||
|
||||
After downloading the images, you can import them into LXD using the following command:
|
||||
```bash
|
||||
lxc image import <path_to_image> --alias <image_alias>
|
||||
```
|
||||
Now you can use the image to create a container:
|
||||
```bash
|
||||
lxc launch <image_alias> <container_name>
|
||||
```
|
||||
|
||||
To log into the container you need to know the automatically generated password. You can get it with the following command:
|
||||
```bash
|
||||
lxc exec <container_name> -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2"
|
||||
```
|
||||
|
||||
If you also want to use Lacus, you can do the same with the Lacus image. After that, you can configure AIL to use Lacus as a crawler. For further information, please refer to the [HOWTO.md](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md).
|
||||
|
||||
## Building the images locally
|
||||
If you want to build the images locally, you can use the `build.sh` script:
|
||||
```bash
|
||||
bash build.sh [OPTIONS]
|
||||
```
|
||||
| Flag | Default Value | Description |
|
||||
| ------------------------------- | ------------- | --------------------------------------------------------------------------------- |
|
||||
| `--ail` | `false` | Activates the creation of the AIL container. |
|
||||
| `--lacus` | `false` | Activates the creation of the Lacus container. |
|
||||
| `--ail-name <container_name>` | `AIL` | Specifies the name of the AIL container. The default is a generic name "AIL". |
|
||||
| `--lacus-name <container_name>` | `Lacus` | Specifies the name of the Lacus container. The default is a generic name "Lacus". |
|
||||
| `-o`, `--outputdir <directory>` | `<none>` | Sets the output directory for the LXD image files. |
|
||||
| `-s`, `--sign` | `false` | Enables the signing of the generated LXD image files. |
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
import json
|
||||
import requests
|
||||
import subprocess
|
||||
import re
|
||||
import os
|
||||
from time import sleep
|
||||
from typing import List, Optional
|
||||
from pathlib import Path
|
||||
|
||||
BUILD_PATH = "/opt/ailbuilder/build"
|
||||
|
||||
class Repo:
|
||||
"""Base class for repository tracking and update checking."""
|
||||
|
||||
def __init__(self, id: str, args: List[str], name: str, outputdir: str) -> None:
|
||||
self.id = id
|
||||
self.args = args
|
||||
self.name = name
|
||||
self.outputdir = outputdir
|
||||
self.last_seen_update = None
|
||||
|
||||
def _check_for_new_update(self) -> bool:
|
||||
latest_update = self._get_latest_update()
|
||||
if latest_update and (latest_update != self.last_seen_update):
|
||||
print(f"New update found for {self.id}")
|
||||
self.last_seen_update = latest_update
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_latest_update(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _save_state(self):
|
||||
try:
|
||||
with open(f'{BUILD_PATH}/systemd/state.json', 'r') as file:
|
||||
states = json.load(file)
|
||||
except FileNotFoundError:
|
||||
states = {}
|
||||
|
||||
states[self.id] = self.last_seen_update
|
||||
|
||||
with open(f'{BUILD_PATH}/systemd/state.json', 'w') as file:
|
||||
json.dump(states, file)
|
||||
|
||||
def load_state(self):
|
||||
try:
|
||||
with open(f'{BUILD_PATH}/systemd/state.json', 'r') as file:
|
||||
states = json.load(file)
|
||||
except FileNotFoundError:
|
||||
states = {}
|
||||
|
||||
self.last_seen_update = states.get(self.id, None)
|
||||
|
||||
def build(self) -> None:
|
||||
if self._check_for_new_update():
|
||||
try:
|
||||
cmd = [f'{BUILD_PATH}/build.sh'] + self.args + ["-o", self.outputdir]
|
||||
print(f"Running {cmd}")
|
||||
result = subprocess.run(cmd, check=False)
|
||||
if result.returncode != 0:
|
||||
print(f"Failed to run {cmd} for {self.id}")
|
||||
return
|
||||
most_recent_dir = max((d for d in Path(self.outputdir).iterdir() if d.is_dir()), key=os.path.getctime, default=None)
|
||||
relative_path = most_recent_dir.relative_to(Path(self.outputdir))
|
||||
if os.path.exists(f"{self.outputdir}/latest_{self.name}"):
|
||||
os.remove(f"{self.outputdir}/latest_{self.name}")
|
||||
os.symlink(relative_path, f"{self.outputdir}/latest_{self.name}")
|
||||
print(f"Created symlink {self.outputdir}/latest_{self.name} to {relative_path}")
|
||||
self._save_state()
|
||||
except Exception as e:
|
||||
print(f"Failed to run {cmd} for {self.id}: {e}")
|
||||
|
||||
class GitHub(Repo):
|
||||
"""Class for tracking GitHub repositories."""
|
||||
|
||||
def __init__(self, id: str, mode: str, args: List[str], name: str, outputdir: str) -> None:
|
||||
super().__init__(id, args, name, outputdir)
|
||||
self.mode = mode
|
||||
|
||||
def _get_latest_update(self) -> Optional[str]:
|
||||
print(f"Fetching {self.mode} for {self.id}")
|
||||
url=f'https://api.github.com/repos/{self.id}/{self.mode}'
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
return response.json()[0]['sha']
|
||||
else:
|
||||
print(f"Failed to fetch {self.mode} for {self.id}")
|
||||
return None
|
||||
|
||||
class APT(Repo):
|
||||
"""Class for tracking APT packages."""
|
||||
|
||||
def __init__(self, id: str, args: List[str], name: str, outputdir: str) -> None:
|
||||
super().__init__(id, args, name, outputdir)
|
||||
|
||||
def _get_latest_update(self) -> Optional[str]:
|
||||
try:
|
||||
cmd = ["apt-cache", "policy", self.id]
|
||||
print (f"Running {cmd}")
|
||||
output = subprocess.check_output(cmd).decode('utf-8')
|
||||
match = re.search(r'Candidate: (\S+)', output)
|
||||
if match:
|
||||
return match.group(1)
|
||||
else:
|
||||
return None
|
||||
except:
|
||||
return None
|
||||
|
||||
def main():
|
||||
with open(f'{BUILD_PATH}/conf/tracker.json') as f:
|
||||
config = json.load(f)
|
||||
|
||||
repos = []
|
||||
for repo in config["github"]:
|
||||
repos.append(GitHub(repo["id"], repo["mode"], repo["args"], repo["name"], config["outputdir"]))
|
||||
|
||||
aptpkg = []
|
||||
for package in config["apt"]:
|
||||
aptpkg.append(APT(package["id"], package["args"], package["name"], config["outputdir"]))
|
||||
|
||||
for repo in repos + aptpkg:
|
||||
if config["sign"]:
|
||||
repo.args.append("-s")
|
||||
repo.load_state()
|
||||
|
||||
while True:
|
||||
for repo in repos:
|
||||
repo.build()
|
||||
for package in aptpkg:
|
||||
package.build()
|
||||
sleep(config["check_interval"])
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,389 @@
|
|||
#!/bin/bash
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
setVars() {
|
||||
DEPEDENCIES=("lxc" "jq")
|
||||
PATH_TO_BUILD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
||||
|
||||
PROJECT_NAME=$(generateName "AIL")
|
||||
STORAGE_POOL_NAME=$(generateName "AIL")
|
||||
NETWORK_NAME=$(generateName "AIL")
|
||||
NETWORK_NAME=${NETWORK_NAME:0:14}
|
||||
|
||||
UBUNTU="ubuntu:22.04"
|
||||
|
||||
AIL_CONTAINER=$(generateName "AIL")
|
||||
LACUS_CONTAINER=$(generateName "LACUS")
|
||||
LACUS_SERVICE_FILE="$PATH_TO_BUILD/conf/lacus.service"
|
||||
}
|
||||
|
||||
setDefaults(){
|
||||
default_ail=false
|
||||
default_ail_image="AIL"
|
||||
default_lacus=false
|
||||
default_lacus_image="Lacus"
|
||||
default_outputdir=""
|
||||
default_sign=false
|
||||
}
|
||||
|
||||
error() {
|
||||
echo -e "${RED}ERROR: $1${NC}"
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo -e "${YELLOW}WARNING: $1${NC}"
|
||||
}
|
||||
|
||||
info() {
|
||||
echo -e "${BLUE}INFO: $1${NC}"
|
||||
}
|
||||
|
||||
success() {
|
||||
echo -e "${GREEN}SUCCESS: $1${NC}"
|
||||
}
|
||||
|
||||
err() {
|
||||
local parent_lineno="$1"
|
||||
local message="$2"
|
||||
local code="${3:-1}"
|
||||
|
||||
if [[ -n "$message" ]] ; then
|
||||
error "Line ${parent_lineno}: ${message}: exiting with status ${code}"
|
||||
else
|
||||
error "Line ${parent_lineno}: exiting with status ${code}"
|
||||
fi
|
||||
|
||||
deleteLXDProject "$PROJECT_NAME"
|
||||
lxc storage delete "$APP_STORAGE"
|
||||
lxc storage delete "$DB_STORAGE"
|
||||
lxc network delete "$NETWORK_NAME"
|
||||
exit "${code}"
|
||||
}
|
||||
|
||||
generateName(){
|
||||
local name="$1"
|
||||
echo "${name}-$(date +%Y%m%d%H%M%S)"
|
||||
}
|
||||
|
||||
|
||||
waitForContainer() {
|
||||
local container_name="$1"
|
||||
|
||||
sleep 3
|
||||
while true; do
|
||||
status=$(lxc list --format=json | jq -e --arg name "$container_name" '.[] | select(.name == $name) | .status')
|
||||
if [ "$status" = "\"Running\"" ]; then
|
||||
echo -e "${BLUE}$container_name ${GREEN}is running.${NC}"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for $container_name container to start."
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
interrupt() {
|
||||
warn "Script interrupted by user. Delete project and exit ..."
|
||||
deleteLXDProject "$PROJECT_NAME"
|
||||
lxc network delete "$NETWORK_NAME"
|
||||
exit 130
|
||||
}
|
||||
|
||||
cleanupProject(){
|
||||
local project="$1"
|
||||
|
||||
info "Starting cleanup ..."
|
||||
echo "Deleting container in project"
|
||||
for container in $(lxc query "/1.0/containers?recursion=1&project=${project}" | jq .[].name -r); do
|
||||
lxc delete --project "${project}" -f "${container}"
|
||||
done
|
||||
|
||||
echo "Deleting images in project"
|
||||
for image in $(lxc query "/1.0/images?recursion=1&project=${project}" | jq .[].fingerprint -r); do
|
||||
lxc image delete --project "${project}" "${image}"
|
||||
done
|
||||
|
||||
echo "Deleting project"
|
||||
lxc project delete "${project}"
|
||||
}
|
||||
|
||||
cleanup(){
|
||||
cleanupProject "$PROJECT_NAME"
|
||||
lxc storage delete "$STORAGE_POOL_NAME"
|
||||
lxc network delete "$NETWORK_NAME"
|
||||
}
|
||||
|
||||
createAILContainer(){
|
||||
lxc launch $UBUNTU "$AIL_CONTAINER" -p default --storage "$STORAGE_POOL_NAME" --network "$NETWORK_NAME"
|
||||
waitForContainer "$AIL_CONTAINER"
|
||||
lxc exec "$AIL_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf
|
||||
lxc exec "$AIL_CONTAINER" -- apt update
|
||||
lxc exec "$AIL_CONTAINER" -- apt upgrade -y
|
||||
lxc exec "$AIL_CONTAINER" -- useradd -m -s /bin/bash ail
|
||||
if lxc exec "$AIL_CONTAINER" -- id ail; then
|
||||
lxc exec "$AIL_CONTAINER" -- usermod -aG sudo ail
|
||||
success "User ail created."
|
||||
else
|
||||
error "User ail not created."
|
||||
exit 1
|
||||
fi
|
||||
lxc exec "$AIL_CONTAINER" -- bash -c "echo 'ail ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/ail"
|
||||
lxc exec "$AIL_CONTAINER" --cwd=/home/ail -- sudo -u ail bash -c "git clone https://github.com/ail-project/ail-framework.git"
|
||||
lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework -- sudo -u ail bash -c "./installing_deps.sh"
|
||||
lxc exec "$AIL_CONTAINER" -- sed -i '/^\[Flask\]/,/^\[/ s/host = 127\.0\.0\.1/host = 0.0.0.0/' /home/ail/ail-framework/configs/core.cfg
|
||||
lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework/bin -- sudo -u ail bash -c "./LAUNCH.sh -l"
|
||||
lxc exec "$AIL_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf
|
||||
}
|
||||
|
||||
createLacusContainer(){
|
||||
lxc launch $UBUNTU "$LACUS_CONTAINER" -p default --storage "$STORAGE_POOL_NAME" --network "$NETWORK_NAME"
|
||||
waitForContainer "$LACUS_CONTAINER"
|
||||
lxc exec "$LACUS_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf
|
||||
lxc exec "$LACUS_CONTAINER" -- apt update
|
||||
lxc exec "$LACUS_CONTAINER" -- apt upgrade -y
|
||||
lxc exec "$LACUS_CONTAINER" -- apt install pipx -y
|
||||
lxc exec "$LACUS_CONTAINER" -- pipx install poetry
|
||||
lxc exec "$LACUS_CONTAINER" -- pipx ensurepath
|
||||
lxc exec "$LACUS_CONTAINER" -- apt install build-essential tcl -y
|
||||
lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/redis/redis.git
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- git checkout 7.2
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make test
|
||||
lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install
|
||||
AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p")
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps"
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env"
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && echo 'no' | /root/.local/bin/poetry run update --init"
|
||||
# Install Tor
|
||||
lxc exec "$LACUS_CONTAINER" -- apt install apt-transport-https -y
|
||||
lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list"
|
||||
lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list"
|
||||
lxc exec "$LACUS_CONTAINER" -- bash -c "wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg > /dev/null"
|
||||
lxc exec "$LACUS_CONTAINER" -- apt update
|
||||
lxc exec "$LACUS_CONTAINER" -- apt install tor deb.torproject.org-keyring -y
|
||||
lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf
|
||||
# Start Lacus
|
||||
lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./config/logging.json.sample ./config/logging.json
|
||||
lxc file push "$LACUS_SERVICE_FILE" "$LACUS_CONTAINER"/etc/systemd/system/lacus.service
|
||||
lxc exec "$LACUS_CONTAINER" -- systemctl daemon-reload
|
||||
lxc exec "$LACUS_CONTAINER" -- systemctl enable lacus.service
|
||||
lxc exec "$LACUS_CONTAINER" -- systemctl start lacus.service
|
||||
}
|
||||
|
||||
createLXDImage() {
|
||||
local container="$1"
|
||||
local image_name="$2"
|
||||
local path_to_repo="$3"
|
||||
local user="$4"
|
||||
|
||||
local commit_id
|
||||
local version
|
||||
commit_id=$(getCommitID "$container" "$path_to_repo")
|
||||
version=$(getVersion "$container" "$path_to_repo" "$user")
|
||||
|
||||
lxc stop "$container"
|
||||
lxc publish "$container" --alias "$image_name"
|
||||
lxc image export "$image_name" "$OUTPUTDIR"
|
||||
local file_name
|
||||
file_name="${image_name}_${version}_${commit_id}.tar.gz"
|
||||
pushd "$OUTPUTDIR" && mv -i "$(ls -t | head -n1)" "$file_name"
|
||||
popd || { error "Failed to rename image file"; exit 1; }
|
||||
sleep 2
|
||||
if $SIGN; then
|
||||
sign "$file_name"
|
||||
fi
|
||||
}
|
||||
|
||||
getCommitID() {
|
||||
local container="$1"
|
||||
local path_to_repo="$2"
|
||||
local current_branch
|
||||
current_branch=$(lxc exec "$container" -- cat "$path_to_repo"/.git/HEAD | awk '{print $2}')
|
||||
local commit_id
|
||||
commit_id=$(lxc exec "$container" -- cat "$path_to_repo"/.git/"$current_branch")
|
||||
echo "$commit_id"
|
||||
}
|
||||
|
||||
getVersion() {
|
||||
local container="$1"
|
||||
local path_to_repo="$2"
|
||||
local user="$3"
|
||||
local version
|
||||
version=$(lxc exec "$container" --cwd="$path_to_repo" -- sudo -u "$user" bash -c "git tag | sort -V | tail -n 1")
|
||||
echo "$version"
|
||||
}
|
||||
|
||||
sign() {
|
||||
if ! command -v gpg &> /dev/null; then
|
||||
error "GPG is not installed. Please install it before running this script with signing."
|
||||
exit 1
|
||||
fi
|
||||
local file=$1
|
||||
SIGN_CONFIG_FILE="$PATH_TO_BUILD/conf/sign.json"
|
||||
|
||||
if [[ ! -f "$SIGN_CONFIG_FILE" ]]; then
|
||||
error "Config file not found: $SIGN_CONFIG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GPG_KEY_ID=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE")
|
||||
GPG_KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE")
|
||||
|
||||
# Check if the GPG key is available
|
||||
if ! gpg --list-keys | grep -q "$GPG_KEY_ID"; then
|
||||
warn "GPG key not found: $GPG_KEY_ID. Create new key."
|
||||
# Setup GPG key
|
||||
KEY_NAME=$(jq -r '.NAME' "$SIGN_CONFIG_FILE")
|
||||
KEY_EMAIL=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE")
|
||||
KEY_COMMENT=$(jq -r '.COMMENT' "$SIGN_CONFIG_FILE")
|
||||
KEY_EXPIRE=$(jq -r '.EXPIRE_DATE' "$SIGN_CONFIG_FILE")
|
||||
KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE")
|
||||
BATCH_FILE=$(mktemp -d)/batch
|
||||
|
||||
cat > "$BATCH_FILE" <<EOF
|
||||
%echo Generating a basic OpenPGP key
|
||||
Key-Type: default
|
||||
Subkey-Type: default
|
||||
Name-Real: ${KEY_NAME}
|
||||
Name-Comment: ${KEY_COMMENT}
|
||||
Name-Email: ${KEY_EMAIL}
|
||||
Expire-Date: ${KEY_EXPIRE}
|
||||
Passphrase: ${KEY_PASSPHRASE}
|
||||
%commit
|
||||
%echo done
|
||||
EOF
|
||||
|
||||
gpg --batch --generate-key "$BATCH_FILE" || { log "Failed to generate GPG key"; exit 1; }
|
||||
rm -r "$BATCH_FILE" || { log "Failed to remove batch file"; exit 1; }
|
||||
fi
|
||||
|
||||
SIGN_DIR="${OUTPUTDIR}/${file/.tar.gz/}"
|
||||
mkdir -p "$SIGN_DIR"
|
||||
|
||||
# Move the file to the new directory
|
||||
mv "${OUTPUTDIR}/${file}" "$SIGN_DIR"
|
||||
|
||||
# Change to the directory
|
||||
pushd "$SIGN_DIR" || exit
|
||||
|
||||
# Signing the file
|
||||
info "Signing file: $file in directory: $SIGN_DIR with key: $GPG_KEY_ID"
|
||||
gpg --default-key "$GPG_KEY_ID" --pinentry-mode loopback --passphrase "$GPG_KEY_PASSPHRASE" --detach-sign "${file}"
|
||||
|
||||
# Check if the signing was successful
|
||||
if [ $? -eq 0 ]; then
|
||||
info "Successfully signed: $file"
|
||||
else
|
||||
error "Failed to sign: $file"
|
||||
exit 1
|
||||
fi
|
||||
popd || exit
|
||||
}
|
||||
|
||||
checkSoftwareDependencies(){
|
||||
|
||||
for dep in "$@"; do
|
||||
if ! command -v "$dep" &> /dev/null; then
|
||||
echo -e "${RED}Error: $dep is not installed.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
}
|
||||
|
||||
# ------------------ MAIN ------------------
|
||||
checkSoftwareDependencies "${DEPEDENCIES[@]}"
|
||||
setVars
|
||||
setDefaults
|
||||
|
||||
VALID_ARGS=$(getopt -o ho:s --long help,outputdir:,sign,ail,lacus,ail-name:,lacus-name: -- "$@")
|
||||
if [[ $? -ne 0 ]]; then
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
eval set -- "$VALID_ARGS"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-h | --help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--ail)
|
||||
ail=true
|
||||
shift
|
||||
;;
|
||||
--lacus)
|
||||
lacus=true
|
||||
shift
|
||||
;;
|
||||
--ail-name)
|
||||
ail_image=$2
|
||||
shift 2
|
||||
;;
|
||||
--lacus-name)
|
||||
lacus_image=$2
|
||||
shift 2
|
||||
;;
|
||||
-o | --outputdir)
|
||||
outputdir=$2
|
||||
shift 2
|
||||
;;
|
||||
-s | --sign)
|
||||
sign=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
AIL=${ail:-$default_ail}
|
||||
LACUS=${lacus:-$default_lacus}
|
||||
AIL_IMAGE=${ail_image:-$default_ail_image}
|
||||
LACUS_IMAGE=${lacus_image:-$default_lacus_image}
|
||||
OUTPUTDIR=${outputdir:-$default_outputdir}
|
||||
SIGN=${sign:-$default_sign}
|
||||
|
||||
if [ ! -e "$OUTPUTDIR" ]; then
|
||||
error "The specified directory does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! $AIL && ! $LACUS; then
|
||||
error "No image specified!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "----------------------------------------"
|
||||
echo "Startting creating LXD images ..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
lxc project create "$PROJECT_NAME"
|
||||
lxc project switch "$PROJECT_NAME"
|
||||
lxc storage create "$STORAGE_POOL_NAME" "dir"
|
||||
lxc network create "$NETWORK_NAME"
|
||||
|
||||
if $AIL; then
|
||||
createAILContainer
|
||||
createLXDImage "$AIL_CONTAINER" "$AIL_IMAGE" "/home/ail/ail-framework" "ail"
|
||||
fi
|
||||
|
||||
if $LACUS; then
|
||||
createLacusContainer
|
||||
createLXDImage "$LACUS_CONTAINER" "$LACUS_IMAGE" "/root/lacus" "root"
|
||||
fi
|
||||
|
||||
echo "----------------------------------------"
|
||||
echo "Build script finished."
|
||||
echo "----------------------------------------"
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[Unit]
|
||||
Description=lacus service
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=root
|
||||
Group=root
|
||||
Type=forking
|
||||
WorkingDirectory=/root/lacus
|
||||
Environment="PATH=/root/.local/bin/poetry:/usr/bin"
|
||||
Environment="LACUS_HOME=/root/lacus"
|
||||
ExecStart=/bin/bash -c "exec /root/.local/bin/poetry run start"
|
||||
ExecStop=/bin/bash -c "exec /root/.local/bin/poetry run stop"
|
||||
StandardOutput=append:/var/log/lacus_message.log
|
||||
StandardError=append:/var/log/lacus_error.log
|
||||
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"NAME": "admin",
|
||||
"EMAIL": "admin@admin.test",
|
||||
"COMMENT": "Key for signing images",
|
||||
"EXPIRE_DATE": 0,
|
||||
"PASSPHRASE": "admin"
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
{
|
||||
"check_interval": 600,
|
||||
"outputdir": "/opt/ailbuilder/images",
|
||||
"sign": true,
|
||||
"github": [
|
||||
{
|
||||
"name": "AIL",
|
||||
"id": "ail-project/ail-framework",
|
||||
"mode": "commits",
|
||||
"args": [
|
||||
"--ail",
|
||||
"--ail-name",
|
||||
"AIL"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Lacus",
|
||||
"id": "ail-project/lacus",
|
||||
"mode": "commits",
|
||||
"args": [
|
||||
"--lacus",
|
||||
"--lacus-name",
|
||||
"Lacus"
|
||||
]
|
||||
}
|
||||
],
|
||||
"apt": []
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
[Unit]
|
||||
Description=Service for building AIL and Lacus LXD images
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=ailbuilder
|
||||
ExecStart=/usr/bin/python3 /opt/ailbuilder/build/ailbuilder.py
|
||||
Restart=on-failure
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
#!/bin/bash
|
||||
|
||||
DIR="$(dirname "$0")"
|
||||
SERVICE_FILE="ailbuilder.service"
|
||||
SERVICE_PATH="/etc/systemd/system/"
|
||||
AILBUILDER_PATH="/opt/ailbuilder"
|
||||
BUILD_DIR="${DIR}/../../build"
|
||||
BATCH_FILE="/tmp/key_batch"
|
||||
SIGN_CONFIG_FILE="../conf/sign.json"
|
||||
|
||||
log() {
|
||||
echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*"
|
||||
}
|
||||
|
||||
echo "Start setting up ailbuilder service ..."
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
log "This script must be run as root or with sudo privileges"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "$AILBUILDER_PATH" ]]; then
|
||||
mkdir -p "$AILBUILDER_PATH"/images || { log "Failed to create directory $AILBUILDER_PATH"; exit 1; }
|
||||
fi
|
||||
|
||||
if [[ -d "$BUILD_DIR" ]]; then
|
||||
cp -r "$BUILD_DIR" "$AILBUILDER_PATH/" || { log "Failed to copy build directory"; exit 1; }
|
||||
else
|
||||
log "Build directory $BUILD_DIR does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create user if it doesn't exist
|
||||
if ! id "ailbuilder" &>/dev/null; then
|
||||
useradd -r -s /bin/false ailbuilder || { log "Failed to create user ailbuilder"; exit 1; }
|
||||
fi
|
||||
|
||||
# Set ownership and permissions
|
||||
chown -R ailbuilder: "$AILBUILDER_PATH" || { log "Failed to change ownership"; exit 1; }
|
||||
chmod -R u+x "$AILBUILDER_PATH/build/"*.py || { log "Failed to set execute permission on scripts"; exit 1; }
|
||||
chmod -R u+x "$AILBUILDER_PATH/build/"*.sh || { log "Failed to set execute permission on scripts"; exit 1; }
|
||||
chmod -R u+w "$AILBUILDER_PATH/images/" || { log "Failed to set execute permission on images dir"; exit 1; }
|
||||
|
||||
# Add user to lxd group
|
||||
sudo usermod -aG lxd ailbuilder || { log "Failed to add user ailbuilder to lxd group"; exit 1; }
|
||||
mkdir -p /home/ailbuilder || { log "Failed to create directory /home/ailbuilder"; exit 1; }
|
||||
chown -R ailbuilder: "/home/ailbuilder" || { log "Failed to change ownership"; exit 1; }
|
||||
chmod -R u+w "/home/ailbuilder" || { log "Failed to set execute permission on home dir"; exit 1; }
|
||||
|
||||
# Setup GPG key
|
||||
KEY_NAME=$(jq -r '.NAME' "$SIGN_CONFIG_FILE")
|
||||
KEY_EMAIL=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE")
|
||||
KEY_COMMENT=$(jq -r '.COMMENT' "$SIGN_CONFIG_FILE")
|
||||
KEY_EXPIRE=$(jq -r '.EXPIRE_DATE' "$SIGN_CONFIG_FILE")
|
||||
KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE")
|
||||
|
||||
if ! sudo -u ailbuilder bash -c "gpg --list-keys | grep -q $KEY_EMAIL"; then
|
||||
cat > "$BATCH_FILE" <<EOF
|
||||
%echo Generating a basic OpenPGP key
|
||||
Key-Type: default
|
||||
Subkey-Type: default
|
||||
Name-Real: ${KEY_NAME}
|
||||
Name-Comment: ${KEY_COMMENT}
|
||||
Name-Email: ${KEY_EMAIL}
|
||||
Expire-Date: ${KEY_EXPIRE}
|
||||
Passphrase: ${KEY_PASSPHRASE}
|
||||
%commit
|
||||
%echo done
|
||||
EOF
|
||||
|
||||
sudo -u ailbuilder gpg --batch --generate-key "$BATCH_FILE" || { log "Failed to generate GPG key"; exit 1; }
|
||||
rm "$BATCH_FILE" || { log "Failed to remove batch file"; exit 1; }
|
||||
fi
|
||||
|
||||
# Copy service file
|
||||
if [[ -f "$SERVICE_FILE" ]]; then
|
||||
cp "${SERVICE_FILE}" "${SERVICE_PATH}" || { log "Failed to copy service file"; exit 1; }
|
||||
else
|
||||
log "Service file $SERVICE_FILE not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Reload systemd, enable and start the service
|
||||
systemctl daemon-reload || { log "Failed to reload systemd daemon"; exit 1; }
|
||||
systemctl enable "${SERVICE_FILE}" || { log "Failed to enable service"; exit 1; }
|
||||
systemctl start "${SERVICE_FILE}" || { log "Failed to start service"; exit 1; }
|
||||
|
||||
log "Service setup completed successfully."
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
#!/bin/bash
|
||||
|
||||
SERVICE_FILE="ailbuilder.service"
|
||||
SERVICE_PATH="/etc/systemd/system/"
|
||||
MISP_AIRGAP_PATH="/opt/ailbuilder"
|
||||
DIR="$(dirname "$0")"
|
||||
BUILD_DIR="${DIR}/../../build"
|
||||
|
||||
|
||||
log() {
|
||||
echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*"
|
||||
}
|
||||
echo "Updating ailbuilder service ..."
|
||||
|
||||
if systemctl is-active --quiet ailbuilder; then
|
||||
systemctl stop ailbuilder || { log "Failed to stop service"; exit 1; }
|
||||
systemctl disable ailbuilder || { log "Failed to disable service"; exit 1; }
|
||||
fi
|
||||
|
||||
if [[ -f "$SERVICE_FILE" ]]; then
|
||||
cp "${SERVICE_FILE}" "${SERVICE_PATH}" || { log "Failed to copy service file"; exit 1; }
|
||||
else
|
||||
log "Service file $SERVICE_FILE not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -d "$BUILD_DIR" ]]; then
|
||||
cp -r "$BUILD_DIR" "$MISP_AIRGAP_PATH/" || { log "Failed to copy build directory"; exit 1; }
|
||||
else
|
||||
log "Build directory $BUILD_DIR does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
systemctl daemon-reload || { log "Failed to reload systemd"; exit 1; }
|
||||
systemctl enable ailbuilder || { log "Failed to enable service"; exit 1; }
|
||||
systemctl start ailbuilder || { log "Failed to start service"; exit 1; }
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[Unit]
|
||||
Description=lacus service
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=root
|
||||
Group=root
|
||||
Type=forking
|
||||
WorkingDirectory=/root/lacus
|
||||
Environment="PATH=/root/.local/bin/poetry:/usr/bin"
|
||||
Environment="LACUS_HOME=/root/lacus"
|
||||
ExecStart=/bin/bash -c "exec /root/.local/bin/poetry run start"
|
||||
ExecStop=/bin/bash -c "exec /root/.local/bin/poetry run stop"
|
||||
StandardOutput=append:/var/log/lacus_message.log
|
||||
StandardError=append:/var/log/lacus_error.log
|
||||
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Loading…
Reference in New Issue