diff --git a/.github/workflows/docker-inner.yml b/.github/workflows/docker-inner.yml index 3a1e5da88..3e99e464a 100644 --- a/.github/workflows/docker-inner.yml +++ b/.github/workflows/docker-inner.yml @@ -3,6 +3,11 @@ name: publish-docker-inner on: workflow_dispatch: inputs: + repo_ref: + required: false + description: 'Set branch or tag or commit id. Default is "main"' + type: string + default: 'main' image_tag: required: true description: 'Set docker image tag. Default is "latest", you can also set "auto" to generate a formated tag, or input a manual tag' @@ -39,7 +44,9 @@ jobs: SCHEDULE_TAG: 'pt29_latest' steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v6 + with: + ref: ${{ github.event.inputs.repo_ref || 'main' }} - name: Parse tag run: | if [[ "${GITHUB_EVENT_NAME}" == 'schedule' ]]; then diff --git a/Dockerfile b/Dockerfile index ffc708d4f..c85b754e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -222,9 +222,8 @@ RUN --mount=type=secret,id=HTTPS_PROXY,env=https_proxy \ pip install fastapi fire openai outlines \ pyzmq aiohttp cloudpickle prometheus_client protobuf numpy pillow einops tiktoken sentencepiece \ partial_json_parser 'ray[default]<3' shortuuid uvicorn pybase64 \ - tilelang==0.1.11 \ 'pydantic>2' openai_harmony dlblas --no-cache-dir -i ${DEFAULT_PYPI_URL} && \ - pip install xgrammar==0.1.32 timm!=1.0.23 --no-cache-dir -i ${DEFAULT_PYPI_URL} --no-deps && \ + pip install xgrammar>=0.1.33 timm!=1.0.23 --no-cache-dir -i ${DEFAULT_PYPI_URL} --no-deps && \ if [ -n "${LMDEPLOY_VERSION}" ]; then \ pip install lmdeploy==${LMDEPLOY_VERSION} --no-deps --no-cache-dir -i ${DEFAULT_PYPI_URL}; \ else \ diff --git a/autotest/module/train.py b/autotest/module/train.py index 1ed84bbac..49da3162f 100644 --- a/autotest/module/train.py +++ b/autotest/module/train.py @@ -33,6 +33,9 @@ def get_cmd(config): ) config["work_dir"] = work_dir + # Print runtime image package list before training for version diagnosis. + image_version_check = 'echo "===== runtime pip list ====="; pip list; ' + # this patch is for torch 2.9.1 Conv3d memory issue fix cudnn_patch = ( "TORCH_VERSION=$(python -c 'import torch;print(torch.__version__.split(chr(43))[0])'); " @@ -41,7 +44,8 @@ def get_cmd(config): if train_type == "sft": command = ( - f"cd {current_dir}; pwd; {pip_package}; export GITHUB_RUN_ID={config.get('run_id')}; export WORK_DIR={work_dir}; " + f"cd {current_dir}; pwd; {pip_package}; {image_version_check}" + f"export GITHUB_RUN_ID={config.get('run_id')}; export WORK_DIR={work_dir}; " + cudnn_patch + f"torchrun --nproc-per-node {nproc_per_node} --master_addr=${{MASTER_ADDR}} --master_port=${{MASTER_PORT}} --nnodes=${{WORLD_SIZE}} --node_rank=${{RANK}} " + f"xtuner/v1/train/cli/{train_type}.py" @@ -66,7 +70,8 @@ def get_cmd(config): infer_type = config.get("parameters", {}).get("infer_backend", "lmdeploy") accelerator = config.get("parameters", {}).get("accelerator", "GPU") command = ( - f"cd {current_dir}; pwd; {pip_package}; export GITHUB_RUN_ID={config.get('run_id')}; export WORK_DIR={work_dir}; " + f"cd {current_dir}; pwd; {pip_package}; {image_version_check}" + f"export GITHUB_RUN_ID={config.get('run_id')}; export WORK_DIR={work_dir}; " + cudnn_patch + f"bash -x autotest/utils/ci_run_rl.sh {accelerator} {infer_type} {config_path} ${{MODEL_PATH}} ${{DATA_PATH}} ${{EVAL_DATA_PATH}}" ) diff --git a/image_build.sh b/image_build.sh index 7fd3374fe..23a0d5bb6 100644 --- a/image_build.sh +++ b/image_build.sh @@ -12,8 +12,8 @@ export CAUSAL_CONV1D_URL=https://github.com/Dao-AILab/causal-conv1d@da6dbaa9fd5a export FLA_URL="${FLA_URL-https://github.com/HAOCHENYE/flash-linear-attention@tmp-tensor-cache}" export TORCH_VERSION=${TORCH_VERSION:-"2.9.1"} -# export LMDEPLOY_VERSION="0.13.0dev" -export LMDEPLOY_URL=https://github.com/InternLM/lmdeploy@efe3b88607756a7ad9411b89627b5ac6ebaa540e +export LMDEPLOY_URL="${LMDEPLOY_URL:-https://github.com/RunningLeon/lmdeploy@c14f389e9b2346f9835954ed90aefc0a0aac7ad4}" +# export LMDEPLOY_VERSION="0.15.0" export PPA_SOURCE="https://mirrors.aliyun.com" export DEFAULT_PYPI_URL=${DEFAULT_PYPI_URL:-"https://mirrors.aliyun.com/pypi/simple"} # mirror https://download.pytorch.org/whl @@ -40,7 +40,7 @@ docker build . \ --build-arg DEEP_GEMM_URL=$DEEP_GEMM_URL \ --build-arg XTUNER_URL=$XTUNER_URL \ --build-arg XTUNER_COMMIT=$XTUNER_COMMIT \ - --build-arg LMDEPLOY_URL=$LMDEPLOY_URL \ + --build-arg LMDEPLOY_URL="$LMDEPLOY_URL" \ --progress=plain \ --label "BASE_IMAGE=$BASE_IMAGE" \ --label "XTUNER_URL=${XTUNER_URL/@/\/tree\/}" \ @@ -53,4 +53,7 @@ docker build . \ --label "DEEP_EP_URL=${DEEP_EP_URL/@/\/tree\/}" \ --label "DEEP_GEMM_URL=${DEEP_GEMM_URL/@/\/tree\/}" \ --label "LMDEPLOY_URL=${LMDEPLOY_URL/@/\/tree\/}" - # --label "LMDEPLOY_VERSION=$LMDEPLOY_VERSION" + +echo "===== pip list in ${image_name}:${image_tag} =====" +docker run --rm --entrypoint pip "${image_name}:${image_tag}" list + diff --git a/pyproject.toml b/pyproject.toml index 925ece261..bb62f7000 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,9 +36,10 @@ dependencies = [ "SentencePiece", "tiktoken", "tilelang==0.1.11", + "apache-tvm-ffi==0.1.11", "torch>=2.6.0", "torchvision", - "transformers==5.2.0", + "transformers==5.14.1", "cyclopts", "transformers_stream_generator", "opencv-python-headless<=4.12.0.88",