Unverified Commit 5f49e74e authored by dmMaze's avatar dmMaze Committed by GitHub
Browse files

Merge pull request #296 from hyrulelinks/dev 

Update shell script to build macos app
parents bc3224f3 08aa247e

README.md

+17 −65
Original line number Diff line number Diff line
@@ -49,83 +49,35 @@ $ python3 launch.py 


如果要使用Sugoi翻译器(仅日译英), 下载[离线模型](https://drive.google.com/drive/folders/1KnDlfUM9zbnYFTo6iCbnBaBKabXfnVJm), 将 "sugoi_translator" 移入BallonsTranslator/ballontranslator/data/models.  



## 构建macOS应用（本方法兼容intel和apple silicon芯片）

## 构建macOS应用（适用apple silicon芯片）

<i>如果构建不成功也可以直接跑源码</i>



![录屏2023-09-11 14 26 49](https://github.com/hyrulelinks/BallonsTranslator/assets/134026642/647c0fa0-ed37-49d6-bbf4-8a8697bc873e)



#### 通过远程脚本一键构建应用

在终端中输入下面的命令自动完成所有构建步骤，由于从github和huggingface下载模型，需要比较好的网络条件

```

curl -L https://raw.githubusercontent.com/dmMaze/BallonsTranslator/dev/scripts/macos-build-script.sh | bash

```



⚠️ 如果网络条件不佳，需要从网盘下载需要的文件，请按照下面的步骤操作

#### 1、准备工作

-   从[MEGA](https://mega.nz/folder/gmhmACoD#dkVlZ2nphOkU5-2ACb5dKw) 或 [Google Drive](https://drive.google.com/drive/folders/1uElIYRLNakJj-YS0Kd3r3HE-wzeEvrWd?usp=sharing)下载`libs`和`models`.

# 第1步：打开终端并确保当前终端窗口的Python大版本号是3.11，可以用下面的命令确认版本号

python3 -V



-  将下载的资源全部放入名为`data`文件夹，最后的目录树结构应该如下所示：



```

data

├── libs

│   └── patchmatch_inpaint.dll

└── models

    ├── aot_inpainter.ckpt

    ├── comictextdetector.pt

    ├── comictextdetector.pt.onnx

    ├── lama_mpe.ckpt

    ├── manga-ocr-base

    │   ├── README.md

    │   ├── config.json

    │   ├── preprocessor_config.json

    │   ├── pytorch_model.bin

    │   ├── special_tokens_map.json

    │   ├── tokenizer_config.json

    │   └── vocab.txt

    ├── mit32px_ocr.ckpt

    ├── mit48pxctc_ocr.ckpt

    └── pkuseg

        ├── postag

        │   ├── features.pkl

        │   └── weights.npz

        ├── postag.zip

        └── spacy_ontonotes

            ├── features.msgpack

            └── weights.npz



7 directories, 23 files

```



-  安装pyenv命令行工具，这是用于管理Python版本的工具

```

# 通过Homebrew途径安装

brew install pyenv

# 第2步：克隆仓库并进入仓库工作目录

git clone -b dev https://github.com/dmMaze/BallonsTranslator.git

cd BallonsTranslator



# 通过官方自动脚本途径安装

curl https://pyenv.run | bash

# 第3步：创建和启用 Python 3.11 虚拟环境

python3 -m venv venv

source venv/bin/activate



# 安装完后需要设置shell环境

echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.zshrc

echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.zshrc

echo 'eval "$(pyenv init -)"' >> ~/.zshrc

```

# 第4步：安装依赖

pip3 install -r requirements.txt



# 第5步：源码运行程序，会自动下载data文件，每个文件在20-400MB左右，合计大约1.67GB，需要比较稳定的网络，如果下载报错，请重复运行下面的命令直至不再下载报错并启动程序

# 如果多次下载失败，也可以从[MEGA](https://mega.nz/folder/gmhmACoD#dkVlZ2nphOkU5-2ACb5dKw) 或 [Google Drive](https://drive.google.com/drive/folders/1uElIYRLNakJj-YS0Kd3r3HE-wzeEvrWd?usp=sharing)下载`libs`和`models`并放入data文件夹

python3 launch.py



#### 2、构建应用

# 第6步：运行下面的远程脚本开始构建macOS应用程序（仅限 Apple Silicon 芯片的 Mac 设备），中途sudo命令需要输入开机密码授予权限

cd ..

curl -L https://raw.githubusercontent.com/dmMaze/BallonsTranslator/dev/scripts/macos-build-script.sh | bash

```

# 进入`data`工作目录

cd data



# 克隆仓库`dev`分支

git clone -b dev https://github.com/dmMaze/BallonsTranslator.git



# 进入`BallonsTranslator`工作目录

cd BallonsTranslator



# 运行构建脚本，运行到pyinstaller环节会要求输入开机密码，输入密码后按下回车即可

sh scripts/build-macos-app.sh

```

> 📌打包好的应用在`./data/BallonsTranslator/dist/BallonsTranslator.app`，将应用拖到macOS的应用程序文件夹即完成安装，开箱即用，不需要另外配置Python环境.  



## 一键翻译

scripts/macos-build-script-arm64.sh

0 → 100644
+144 −0
Original line number Diff line number Diff line 
# Clone repository

echo "STEP 1: Clone repository."

git clone -b dev https://github.com/dmMaze/BallonsTranslator.git

cd BallonsTranslator



# Define directories

DATA_DIR='data'

LIBS_DIR='data/libs'

MODELS_DIR='data/models'

MANGA_OCR_BASE_DIR='data/models/manga-ocr-base'

PKUSEG_DIR='data/models/pkuseg'

POSTAG_DIR='data/models/pkuseg/postag'

SPACY_ONTONOTES_DIR='data/models/pkuseg/spacy_ontonotes'



# Check and make directories

mkdir -p "$DATA_DIR"

mkdir -p "$LIBS_DIR"

mkdir -p "$MODELS_DIR" 

mkdir -p "$MANGA_OCR_BASE_DIR"

mkdir -p "$PKUSEG_DIR"

mkdir -p "$POSTAG_DIR"

mkdir -p "$SPACY_ONTONOTES_DIR"



# Create and activate Python virtual environment

echo "STEP 2: Create and activate Python virtual environment"

python_version=$(python3 -V 2>&1 | cut -d" " -f2 | cut -d"." -f1-2)



if ! which python3 >/dev/null 2>&1; then

    echo "ERROR: ❌ The 'python3' command not found."

    echo "ERROR: Please check the Python environment configuration."

    exit 1

else

    echo "INFO: The 'python3' command found." 

    if [ "$python_version" == "3.11" ]; then

        echo "INFO: ✅ The current Python version is 3.11"

        python3 -m venv venv

        echo "INFO: ✅ Python virtual enviroment created."

        source venv/bin/activate

        echo "INFO: ✅ Python virtual enviroment activated."

    else

        echo "ERROR: ❌ The current Python version is $python_version but 3.11 is required."

        echo "ERROR: Please switch to Python 3.11 before running this script."

        exit 1

    fi

fi



# Check data file hash

echo "STEP 4: Check data file hash."



# Function to calculate file hash

calculate_hash() {

    local file_path=$1

    shasum -a 256 "$file_path" | cut -d ' ' -f 1

}



# Function to check file hash

check_file_hash() {

    local files=(

        'alphabet-all-v5.txt|data|c1295ae1962e69e35b5b225a0405d1f3432e368c9941d23bfd3acda12654da33'

        'alphabet-all-v7.txt|data|f5722368146aa0fbcc9f4726866e4efc3203318ebb66c811d8cbbe915576538a'

        'macos_libopencv_world.4.8.0.dylib|data/libs|843704ab096d3afd8709abe2a2c525ce3a836bb0a629ed1ee9b8f5cee9938310'

        'macos_libpatchmatch_inpaint.dylib|data/libs|849ca84759385d410c9587d69690e668822a3fc376ce2219e583e7e0be5b5e9a'

        'aot_inpainter.ckpt|data/models|878d541c68648969bc1b042a6e997f3a58e49b6c07c5636ad55130736977149f'

        'comictextdetector.pt|data/models|1f90fa60aeeb1eb82e2ac1167a66bf139a8a61b8780acd351ead55268540cccb'

        'comictextdetector.pt.onnx|data/models|1a86ace74961413cbd650002e7bb4dcec4980ffa21b2f19b86933372071d718f'

        'lama_large_512px.ckpt|data/models|11d30fbb3000fb2eceae318b75d9ced9229d99ae990a7f8b3ac35c8d31f2c935'

        'lama_mpe.ckpt|data/models|d625aa1b3e0d0408acfd6928aa84f005867aa8dbb9162480346a4e20660786cc'

        'config.json|data/models/manga-ocr-base|8c0e395de8fa699daaac21aee33a4ba9bd1309cfbff03147813d2a025f39f349'

        'preprocessor_config.json|data/models/manga-ocr-base|af4eb4d79cf61b47010fc0bc9352ee967579c417423b4917188d809b7e048948'

        'pytorch_model.bin|data/models/manga-ocr-base|c63e0bb5b3ff798c5991de18a8e0956c7ee6d1563aca6729029815eda6f5c2eb'

        'README.md|data/models/manga-ocr-base|32f413afcc4295151e77d25202c5c5d81ef621b46f947da1c3bde13256dc0d5f'

        'special_tokens_map.json|data/models/manga-ocr-base|303df45a03609e4ead04bc3dc1536d0ab19b5358db685b6f3da123d05ec200e3'

        'tokenizer_config.json|data/models/manga-ocr-base|d775ad1deac162dc56b84e9b8638f95ed8a1f263d0f56f4f40834e26e205e266'

        'vocab.txt|data/models/manga-ocr-base|344fbb6b8bf18c57839e924e2c9365434697e0227fac00b88bb4899b78aa594d'

        'mit32px_ocr.ckpt|data/models|d9f619a9dccce8ce88357d1b17d25f07806f225c033ea42c64e86c45446cfe71'

        'mit48pxctc_ocr.ckpt|data/models|8b0837a24da5fde96c23ca47bb7abd590cd5b185c307e348c6e0b7238178ed89'

        'ocr_ar_48px.ckpt|data/models|29daa46d080818bb4ab239a518a88338cbccff8f901bef8c9db191a7cb97671d'

        'features.pkl|data/models/pkuseg/postag|17d734c186a0f6e76d15f4990e766a00eed5f72bea099575df23677435ee749d'

        'weights.npz|data/models/pkuseg/postag|2bbd53b366be82a1becedb4d29f76296b36ad7560b6a8c85d54054900336d59a'

        'features.msgpack|data/models/pkuseg/spacy_ontonotes|fd4322482a7018b9bce9216173ae9d2848efe6d310b468bbb4383fb55c874a18'

        'weights.npz|data/models/pkuseg/spacy_ontonotes|5ada075eb25a854f71d6e6fa4e7d55e7be0ae049255b1f8f19d05c13b1b68c9e'

        'pkusegscores.json|data|ca6b8c6b8ba70d4370b0f2de6bd128ebb0f5f64ff06f01ba6358e49a776b0c3f'

    )

        

    # Iterate through file information

    for file_info in "${files[@]}"; do

        IFS='|' read -r -a file_data <<< "$file_info"

        target_file="${file_data[0]}"

        target_dir="${file_data[1]}"

        target_precalculated_hash="${file_data[2]}"

        target_file_path="$target_dir/$target_file"



        # Check if $target_file exists

        if [ -e "$target_file_path" ]; then

            target_computed_hash=$(calculate_hash "$target_file_path")

            

            # Compare hashes

            if [ "$target_computed_hash" == "$target_precalculated_hash" ]; then

                echo "INFO: ✅ $target_file found and hash matches."

            else

                echo "WARNING: ❌ $target_file found but hash mismatches."

                echo "INFO: Expected hash: $target_precalculated_hash"

                echo "INFO: Computed hash: $target_computed_hash"

                exit 1

            fi

        else

            echo "WARNING: ❌ $target_file not found at $target_file_path."

            exit 1

        fi

    done

}



# Call functions

check_file_hash



# Install Python dependencies

echo "STEP 6: Install Python dependencies."

pip3 install -r requirements.txt

pip3 install pyinstaller



# Delete .DS_Store files 

echo "STEP 7: Delete .DS_Store files."

echo "INFO: Permission required to delete .DS_Store files."

sudo find ./ -name '.DS_Store'

sudo find ./ -name '.DS_Store' -delete

echo "INFO: ✅ .DS_Store files all deleted."



# Create packaged app

echo "STEP 8: Create packaged app."

echo "INFO: Use the pyinstaller spec file to bundle the app."

sudo pyinstaller launch.spec



# Check if app exists

app_path="dist/BallonsTranslator.app"

if [ -e "$app_path" ]; then

    # Copy app to Downloads folder

    echo "INFO: Copying app to Downloads folder..."

    ditto "$app_path" "$HOME/Downloads/BallonsTranslator.app"

    echo "INFO: ✅ The app is now in your Downloads folder."

    echo "INFO: Drag and drop the app icon into Applications folder to install it."

    open $HOME/Downloads

else

    echo "ERROR: ❌ App not found. Please build the app first."

fi

scripts/macos-build-script.sh

deleted100644 → 0
+0 −260
Original line number Diff line number Diff line 
# Clone repository

echo "STEP 1: Clone repository."

git clone -b dev https://github.com/dmMaze/BallonsTranslator.git

cd BallonsTranslator



# Define directories

DATA_DIR='data'

LIBS_DIR='data/libs'

MODELS_DIR='data/models'

MANGA_OCR_BASE_DIR='data/models/manga-ocr-base'

PKUSEG_DIR='data/models/pkuseg'

POSTAG_DIR='data/models/pkuseg/postag'

SPACY_ONTONOTES_DIR='data/models/pkuseg/spacy_ontonotes'



# Check and make directories

mkdir -p "$DATA_DIR"

mkdir -p "$LIBS_DIR"

mkdir -p "$MODELS_DIR" 

mkdir -p "$MANGA_OCR_BASE_DIR"

mkdir -p "$PKUSEG_DIR"

mkdir -p "$POSTAG_DIR"

mkdir -p "$SPACY_ONTONOTES_DIR"



# Create and activate Python virtual environment

echo "STEP 2: Create and activate Python virtual environment"

python_version=$(python3 -V 2>&1 | cut -d" " -f2 | cut -d"." -f1-2)



if ! which python3 >/dev/null 2>&1; then

    echo "ERROR: ❌ The 'python3' command not found."

    echo "ERROR: Please check the Python environment configuration."

    exit 1

else

    echo "INFO: The 'python3' command found." 

    if [ "$python_version" == "3.11" ]; then

        echo "INFO: ✅ The current Python version is 3.11"

        python3 -m venv venv

        echo "INFO: ✅ Python virtual enviroment created."

        source venv/bin/activate

        echo "INFO: ✅ Python virtual enviroment activated."

    else

        echo "ERROR: ❌ The current Python version is $python_version but 3.11 is required."

        echo "ERROR: Please switch to Python 3.11 before running this script."

        exit 1

    fi

fi



# OpenCV installation check

echo "STEP 3: Check installation of OpenCV."

echo "INFO: Install OpenCV Python package in virtual environment."

pip3 install opencv-python

echo "INFO: Checking OpenCV installation..."

python3 -c "import cv2" 2>/dev/null

if [ $? -eq 0 ]; then

    opencv_version=$(python3 -c "import cv2; print(cv2.__version__)")

    echo "INFO: ✅ OpenCV is installed. Version: $opencv_version"

else

    echo "ERROR: ❌ OpenCV is not installed."

    echo "ERROR: Please install OpenCV before running this script."

    echo "INFO: Recommand install via Homebrew with command 'brew install opencv'."

    exit 1

fi



# Download extra data files

echo "STEP 4: Download data files."



# Function to calculate file hash

calculate_hash() {

    local file_path=$1

    shasum -a 256 "$file_path" | cut -d ' ' -f 1

}



# Function to download and process files

download_and_process_files() {

    local files=(

        'postag.zip|https://github.com/lancopku/pkuseg-python/releases/download/v0.0.16|zip|features.pkl|features.pkl|data/models/pkuseg/postag|17d734c186a0f6e76d15f4990e766a00eed5f72bea099575df23677435ee749d'

        'postag.zip|https://github.com/lancopku/pkuseg-python/releases/download/v0.0.16|zip|weights.npz|weights.npz|data/models/pkuseg/postag|2bbd53b366be82a1becedb4d29f76296b36ad7560b6a8c85d54054900336d59a'

        'spacy_ontonotes.zip|https://github.com/explosion/spacy-pkuseg/releases/download/v0.0.26|zip|features.msgpack|features.msgpack|data/models/pkuseg/spacy_ontonotes|fd4322482a7018b9bce9216173ae9d2848efe6d310b468bbb4383fb55c874a18'

        'spacy_ontonotes.zip|https://github.com/explosion/spacy-pkuseg/releases/download/v0.0.26|zip|weights.npz|weights.npz|data/models/pkuseg/spacy_ontonotes|5ada075eb25a854f71d6e6fa4e7d55e7be0ae049255b1f8f19d05c13b1b68c9e'

        )

        

    # Iterate through file information

    for file_info in "${files[@]}"; do

        IFS='|' read -r -a file_data <<< "$file_info"

        source_file="${file_data[0]}"

        source_file_base_url="${file_data[1]}"

        is_zip="${file_data[2]}"

        unzip_file="${file_data[3]}"

        target_file="${file_data[4]}"

        target_dir="${file_data[5]}"

        target_file_expected_hash="${file_data[6]}"

        

        # Combine source file and base URL to get download URL

        local download_url="$source_file_base_url/$source_file"

        

        # Check if target_file exists and verify hash if it does

        if [ -e "$target_dir/$target_file" ]; then

            echo "INFO: $target_file already exists, verifying hash..."

            computed_hash=$(calculate_hash "$target_dir/$target_file")

            if [ "$computed_hash" == "$target_file_expected_hash" ]; then

                echo "INFO: ✅ Existing $target_file hash verification passed."

                continue

            else

                echo "WARNING: ❌ Existing $target_file hash verification failed."

                rm -rf "$target_dir/$target_file"

            fi

        fi

            

        # Download and process accordingly based on is_zip and unzip_file

        echo "INFO: Downloading $target_file..."

        if [[ "$is_zip" == "zip" ]]; then

            curl -L "$download_url" -o "$source_file"

            unzip -j "$source_file" "$unzip_file" -d "$target_dir"

            if [ "$unzip_file" != "$target_file" ]; then

                # Rename the file

                mv "$target_dir/$unzip_file" "$target_dir/$target_file"

            fi

            rm -rf "$source_file"

        else

            curl -L "$download_url" -o "$target_dir/$target_file"

        fi

        

        # Calculate hash after download and processing

        downloaded_file_hash=$(calculate_hash "$target_dir/$target_file")

    

        # Check if hash matches expected hash

        if [ "$downloaded_file_hash" == "$target_file_expected_hash" ]; then

            echo "INFO: ✅ Downloaded $target_file hash verification passed."

            continue

        else

            echo "WARNING: ❌ Downloaded $target_file hash verification failed."

            # Remove the existing file

            rm -f "$target_dir/$target_file"



            # Redownload the file

            if [[ "$is_zip" == "zip" ]]; then

                curl -L "$download_url" -o "$source_file"

                unzip -j "$source_file" "$unzip_file" -d "$target_dir"

                if [ "$unzip_file" != "$target_file" ]; then

                    mv "$target_dir/$unzip_file" "$target_dir/$target_file"

                fi

                rm -f "$source_file"

            else

                curl -L "$download_url" -o "$target_dir/$target_file"

            fi

            

            # Calculate hash after re-download

            redownloaded_file_hash=$(calculate_hash "$target_dir/$target_file")

            

            # Check if hash matches expected hash after re-download

            if [ "$redownloaded_file_hash" == "$target_file_expected_hash" ]; then

                echo "INFO: ✅ Re-downloaded $target_file hash verification passed."

                continue

            else

                echo "WARNING: ❌ Re-downloaded $target_file hash verification failed."

                echo "ERROR: ❌ Unable to download $target_file. Exiting."

                exit 1

            fi

        fi

    done

}



# Function to thin libraries based on system architecture

thin_liarary_files() {

    local arch=$(uname -m)

    

    # Thin multi-architecture library files into compatible single arch libraries

    echo "INFO: System architecture is $arch."

    echo "INFO: Extracting architecture specific libraries..."

    if [ "$arch" = "arm64" ]; then

        ditto --arch arm64 "$LIBS_DIR/libopencv_world.4.4.0.dylib" "$LIBS_DIR/libopencv_world2.4.4.0.dylib"

        ditto --arch arm64 "$LIBS_DIR/libpatchmatch_inpaint.dylib" "$LIBS_DIR/libpatchmatch_inpaint2.dylib"

    else

        ditto --arch x86_64 "$LIBS_DIR/libopencv_world.4.4.0.dylib" "$LIBS_DIR/libopencv_world2.4.4.0.dylib"

        ditto --arch x86_64 "$LIBS_DIR/libpatchmatch_inpaint.dylib" "$LIBS_DIR/libpatchmatch_inpaint2.dylib"

    fi

    

    # Remove fat libraries

    rm "$LIBS_DIR/libopencv_world.4.4.0.dylib" "$LIBS_DIR/libpatchmatch_inpaint.dylib"

    mv "$LIBS_DIR/libopencv_world2.4.4.0.dylib" "$LIBS_DIR/libopencv_world.4.4.0.dylib"

    mv "$LIBS_DIR/libpatchmatch_inpaint2.dylib" "$LIBS_DIR/libpatchmatch_inpaint.dylib"

    

    echo "INFO: ✅ Single architecture library files generated."

}



# Call the download functions

download_and_process_files

thin_liarary_files



# Checklist of extra data files

check_list="

data/alphabet-all-v5.txt

$LIBS_DIR/libopencv_world.4.4.0.dylib

$LIBS_DIR/libpatchmatch_inpaint.dylib

$MODELS_DIR/aot_inpainter.ckpt

$MODELS_DIR/comictextdetector.pt

$MODELS_DIR/comictextdetector.pt.onnx

$MODELS_DIR/lama_mpe.ckpt

$MANGA_OCR_BASE_DIR/README.md

$MANGA_OCR_BASE_DIR/config.json

$MANGA_OCR_BASE_DIR/preprocessor_config.json

$MANGA_OCR_BASE_DIR/pytorch_model.bin

$MANGA_OCR_BASE_DIR/special_tokens_map.json

$MANGA_OCR_BASE_DIR/tokenizer_config.json

$MANGA_OCR_BASE_DIR/vocab.txt

$MODELS_DIR/mit32px_ocr.ckpt

$MODELS_DIR/mit48pxctc_ocr.ckpt

$POSTAG_DIR/features.pkl

$POSTAG_DIR/weights.npz

$SPACY_ONTONOTES_DIR

$SPACY_ONTONOTES_DIR/features.msgpack

$SPACY_ONTONOTES_DIR/weights.npz

data/pkusegscores.json

"



# Validate extra data files exist

echo "STEP 5: Validate data files exist."

fail=false

for item in $check_list; do

    if [ ! -e "$item" ]; then

        echo "ERROR: ❌ $item not found"

        fail=true

    fi

done

 

if [ "$fail" = true ]; then

    echo "ERROR: ❌ Data files check failed. Exiting."

    exit 1

else

    echo "INFO: ✅ Data files all exist."

fi



# Install Python dependencies

echo "STEP 6: Install Python dependencies."

pip3 install -r requirements.txt

pip3 install pyinstaller



# Delete .DS_Store files 

echo "STEP 7: Delete .DS_Store files."

echo "INFO: Permission required to delete .DS_Store files."

sudo find ./ -name '.DS_Store'

sudo find ./ -name '.DS_Store' -delete

echo "INFO: ✅ .DS_Store files all deleted."



# Create packaged app

echo "STEP 8: Create packaged app."

echo "INFO: Use the pyinstaller spec file to bundle the app."

sudo pyinstaller launch.spec



# Check if app exists

app_path="dist/BallonsTranslator.app"

if [ -e "$app_path" ]; then

    # Copy app to Downloads folder

    echo "INFO: Copying app to Downloads folder..."

    ditto "$app_path" "$HOME/Downloads/BallonsTranslator.app"

    echo "INFO: ✅ The app is now in your Downloads folder."

    echo "INFO: Drag and drop the app icon into Applications folder to install it."

    open $HOME/Downloads

else

    echo "ERROR: ❌ App not found. Please build the app first."

fi