+ REPO=/workspace/verl + cd /workspace/verl + export MLFLOW_TRACKING_URI=http://:5000/mlflow + MLFLOW_TRACKING_URI=http://:5000/mlflow + export MODEL_PATH=/models/SmolLM2-135M-Instruct + MODEL_PATH=/models/SmolLM2-135M-Instruct + export TRAIN_FILE=/data/char_count/rl/train.parquet + TRAIN_FILE=/data/char_count/rl/train.parquet + export TEST_FILE=/data/char_count/rl/test.parquet + TEST_FILE=/data/char_count/rl/test.parquet + export NGPUS_PER_NODE=1 + NGPUS_PER_NODE=1 + export NNODES=1 + NNODES=1 + export ROLLOUT_TP=1 + ROLLOUT_TP=1 + export TRAIN_BATCH_SIZE=128 + TRAIN_BATCH_SIZE=128 + export PPO_MINI_BATCH_SIZE=16 + PPO_MINI_BATCH_SIZE=16 + export PPO_MICRO_BATCH_SIZE_PER_GPU=8 + PPO_MICRO_BATCH_SIZE_PER_GPU=8 + export LOG_PROB_MICRO_BATCH_SIZE_PER_GPU=8 + LOG_PROB_MICRO_BATCH_SIZE_PER_GPU=8 + export MAX_PROMPT_LENGTH=128 + MAX_PROMPT_LENGTH=128 + export MAX_RESPONSE_LENGTH=128 + MAX_RESPONSE_LENGTH=128 + export ACTOR_LR=1e-6 + ACTOR_LR=1e-6 + export ENTROPY_COEFF=0 + ENTROPY_COEFF=0 + export ROLLOUT_N=8 + ROLLOUT_N=8 + export ROLLOUT_GPU_MEM_UTIL=0.7 + ROLLOUT_GPU_MEM_UTIL=0.7 + export PROJECT_NAME=verl_char_count + PROJECT_NAME=verl_char_count + export EXPERIMENT_NAME=char-count-smol135m-instruct-grpo-direct-1gpu + EXPERIMENT_NAME=char-count-smol135m-instruct-grpo-direct-1gpu + export SAVE_FREQ=-1 + SAVE_FREQ=-1 + export TEST_FREQ=5 + TEST_FREQ=5 + export TOTAL_EPOCHS=5 + TOTAL_EPOCHS=5 + uv sync --frozen --all-packages --extra vllm --extra fsdp Using CPython 3.12.3 interpreter at: /usr/bin/python3 Creating virtual environment at: .venv Building verl @ file:///workspace/verl Built verl @ file:///workspace/verl Prepared 1 package in 397ms Installed 252 packages in 4.31s + absl-py==2.4.0 + accelerate==1.14.0 + aiohappyeyeballs==2.6.2 + aiohttp==3.14.1 + aiohttp-cors==0.8.1 + aiosignal==1.4.0 + annotated-doc==0.0.4 + annotated-types==0.7.0 + anthropic==0.109.1 + antlr4-python3-runtime==4.9.3 + anyio==4.13.0 + apache-tvm-ffi==0.1.9 + astor==0.8.1 + attrs==26.1.0 + av==17.1.0 + blake3==1.0.8 + cachetools==7.1.4 + cbor2==6.1.2 + certifi==2026.5.20 + cffi==2.0.0 + charset-normalizer==3.4.7 + click==8.4.1 + cloudpickle==3.1.2 + codetiming==1.4.0 + colorful==0.5.8 + compressed-tensors==0.17.0 + cryptography==49.0.0 + cuda-bindings==13.3.1 + cuda-core==1.0.1 + cuda-pathfinder==1.5.5 + cuda-python==13.3.1 + cuda-tile==1.3.0 + cuda-toolkit==13.0.2 + datasets==5.0.0 + depyf==0.20.0 + detect-installer==0.1.0 + dill==0.4.1 + diskcache==5.6.3 + distlib==0.4.3 + distro==1.9.0 + dnspython==2.8.0 + docstring-parser==0.18.0 + einops==0.8.2 + email-validator==2.3.0 + fastapi==0.136.3 + fastapi-cli==0.0.24 + fastapi-cloud-cli==0.20.0 + fastar==0.11.0 + fastsafetensors==0.3.2 + filelock==3.29.4 + flash-attn==2.8.3 + flashinfer-cubin==0.6.12 + flashinfer-python==0.6.12 + frozenlist==1.8.0 + fsspec==2026.4.0 + gitdb==4.0.12 + gitpython==3.1.50 + google-api-core==2.31.0 + google-auth==2.54.0 + googleapis-common-protos==1.75.0 + grpcio==1.81.1 + h11==0.16.0 + hf-xet==1.5.1 + httpcore==1.0.9 + httptools==0.8.0 + httpx==0.28.1 + httpx-sse==0.4.3 + huggingface-hub==1.19.0 + humming-kernels==0.1.6 + hydra-core==1.3.3 + idna==3.18 + ijson==3.5.0 + importlib-metadata==9.0.0 + iniconfig==2.3.0 + interegular==0.3.3 + jinja2==3.1.6 + jiter==0.15.0 + jmespath==1.1.0 + jsonschema==4.26.0 + jsonschema-specifications==2025.9.1 + lark==1.2.2 + liger-kernel==0.8.0 + llguidance==1.7.6 + llvmlite==0.47.0 + lm-format-enforcer==0.11.3 + loguru==0.7.3 + markdown==3.10.2 + markdown-it-py==4.2.0 + markupsafe==3.0.3 + mathruler==0.1.0 + mbridge==0.15.1 (from git+https://github.com/ISEEKYAN/mbridge.git@641a5a01de71080b2200d10e369090e40c9a351c) + mcp==1.27.2 + mdurl==0.1.2 + mistral-common==1.11.3 + ml-dtypes==0.5.4 + model-hosting-container-standards==0.1.15 + mpmath==1.3.0 + msgpack==1.2.0 + msgspec==0.21.1 + multidict==6.7.1 + multiprocess==0.70.19 + networkx==3.6.1 + ninja==1.13.0 + numba==0.65.0 + numpy==2.4.6 + nvidia-cublas== + nvidia-cuda-cccl==.1 + nvidia-cuda-crt==13.3.73 + nvidia-cuda-cupti==13.0.85 + nvidia-cuda-nvcc==13.2.86 + nvidia-cuda-nvrtc==13.0.88 + nvidia-cuda-runtime==13.0.96 + nvidia-cuda-tileiras==13.2.86 + nvidia-cudnn-cu13== + nvidia-cudnn-frontend==1.26.0 + nvidia-cufft== + nvidia-cufile== + nvidia-curand== + nvidia-cusolver== + nvidia-cusparse== + nvidia-cusparselt-cu13==0.8.0 + nvidia-cutlass-dsl==4.5.2 + nvidia-cutlass-dsl-libs-base==4.5.2 + nvidia-cutlass-dsl-libs-cu13==4.5.2 + nvidia-ml-py==13.610.43 + nvidia-nccl-cu13==2.28.9 + nvidia-nvjitlink==13.0.88 + nvidia-nvshmem-cu13==3.4.5 + nvidia-nvtx==13.0.85 + nvidia-nvvm==13.2.86 + nvtx==0.2.15 + omegaconf==2.3.1 + openai==2.41.1 + openai-harmony==0.0.8 + opencensus==0.11.4 + opencensus-context==0.1.3 + opencv-python-headless== + opentelemetry-api==1.42.1 + opentelemetry-exporter-otlp==1.42.1 + opentelemetry-exporter-otlp-proto-common==1.42.1 + opentelemetry-exporter-otlp-proto-grpc==1.42.1 + opentelemetry-exporter-otlp-proto-http==1.42.1 + opentelemetry-exporter-prometheus==0.63b1 + opentelemetry-proto==1.42.1 + opentelemetry-sdk==1.42.1 + opentelemetry-semantic-conventions==0.63b1 + opentelemetry-semantic-conventions-ai==0.5.1 + orjson==3.11.9 + outlines-core==0.2.14 + packaging==25.0 + pandas==3.0.3 + partial-json-parser==.post7 + peft==0.19.1 + pillow==12.2.0 + platformdirs==4.10.0 + pluggy==1.6.0 + prometheus-client==0.25.0 + prometheus-fastapi-instrumentator==8.0.0 + propcache==0.5.2 + proto-plus==1.28.0 + protobuf==6.33.6 + psutil==7.2.2 + py-cpuinfo==9.0.0 + py-spy==0.4.2 + pyarrow==24.0.0 + pyasn1==0.6.3 + pyasn1-modules==0.4.2 + pybase64==1.4.3 + pybind11==3.0.4 + pycountry==26.2.16 + pycparser==3.0 + pydantic==2.13.4 + pydantic-core==2.46.4 + pydantic-extra-types==2.11.1 + pydantic-settings==2.14.1 + pyelftools==0.33 + pygments==2.20.0 + pyjwt==2.13.0 + pylatexenc==2.10 + pytest==9.1.0 + pytest-asyncio==1.4.0 + pytest-rerunfailures==16.3 + python-dateutil==2.9.0.post0 + python-discovery==1.4.2 + python-dotenv==1.2.2 + python-json-logger==4.1.0 + python-multipart==0.0.32 + pyvers==0.1.0 + pyyaml==6.0.3 + pyzmq==27.1.0 + quack-kernels==0.5.0 + qwen-vl-utils==0.0.14 + ray==2.55.1 + referencing==0.37.0 + regex==2026.5.9 + requests==2.34.2 + rich==15.0.0 + rich-toolkit==0.20.1 + rignore==0.7.6 + rpds-py==2026.5.1 + safetensors==0.8.0 + sentencepiece==0.2.1 + sentry-sdk==2.62.0 + setproctitle==1.3.7 + setuptools==80.10.2 + shellingham==1.5.4 + six==1.17.0 + smart-open==7.6.1 + smmap==5.0.3 + sniffio==1.3.1 + sse-starlette==3.4.4 + starlette==1.3.1 + supervisor==4.3.0 + sympy==1.14.0 + tabulate==0.10.0 + tensorboard==2.20.0 + tensorboard-data-server==0.7.2 + tensordict==0.10.0 + tiktoken==0.13.0 + tilelang==0.1.9 + tokenizers==0.22.2 + tokenspeed-mla==0.1.2 + tokenspeed-triton==3.7.10.post20260531 + torch==2.11.0+cu130 + torch-c-dlpack-ext==0.1.5 + torchaudio==2.11.0+cu130 + torchdata==0.11.0 + torchvision==0.26.0+cu130 + tqdm==4.68.2 + transferqueue==0.1.9.dev0 (from git+https://github.com/Ascend/TransferQueue.git@434f8c476b4be24bc087e6e95070e64efcc739f9) + transformers==5.5.3 + triton==3.6.0 + trl==0.27.0 + typer==0.25.1 + typing-extensions==4.15.0 + typing-inspection==0.4.2 + urllib3==2.7.0 + uvicorn==0.49.0 + uvloop==0.22.1 + verl==0.9.0.dev0 (from file:///workspace/verl) + virtualenv==21.5.0 + vllm==0.24.0 + wandb==0.27.2 + watchfiles==1.2.0 + websockets==16.0 + werkzeug==3.1.8 + wrapt==2.2.1 + xgrammar==0.2.2 + xxhash==3.7.0 + yarl==1.24.2 + z3-solver== + zipp==4.1.0 + '[' '!' -f /models/SmolLM2-135M-Instruct/config.json ']' + '[' '!' -f /data/char_count/rl/train.parquet ']' + '[' '!' -f /data/char_count/rl/test.parquet ']' + UV_OFFLINE=0 + uv pip install --python .venv/bin/python -q mlflow==3.17.2 + .venv/bin/python -c 'import mlflow, pandas; print(f'\''env ok: mlflow {mlflow.__version__} | pandas {pandas.__version__}'\'')' env ok: mlflow 3.17.2 | pandas 3.0.3 + export UV_NO_SYNC=1 + UV_NO_SYNC=1 + exec bash ./examples/grpo_trainer/run_qwen3_4b_fsdp.sh actor_rollout_ref.actor.strategy=fsdp2 actor_rollout_ref.actor.fsdp_config.param_offload=True actor_rollout_ref.actor.fsdp_config.optimizer_offload=True actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 actor_rollout_ref.actor.use_kl_loss=False actor_rollout_ref.actor.kl_loss_coef=0.0 actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8192 actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=8192 actor_rollout_ref.rollout.enforce_eager=True reward.custom_reward_function.path=/run/reward_tolerant.py reward.custom_reward_function.name=compute_score ++trainer.val_before_train=True 'trainer.logger=["console","mlflow"]' trainer.default_local_dir=/checkpoints/char-count-smol135m ++ python3 -c 'import torch_npu' ++ echo gpu + DEVICE=gpu + INFER_BACKEND=vllm + MODEL_PATH=/models/SmolLM2-135M-Instruct + TRAIN_FILE=/data/char_count/rl/train.parquet + TEST_FILE=/data/char_count/rl/test.parquet + NNODES=1 + NGPUS_PER_NODE=1 + TRAIN_BATCH_SIZE=128 + PPO_MINI_BATCH_SIZE=16 + PPO_MICRO_BATCH_SIZE_PER_GPU=8 + LOG_PROB_MICRO_BATCH_SIZE_PER_GPU=8 + MAX_PROMPT_LENGTH=128 + MAX_RESPONSE_LENGTH=128 + ACTOR_LR=1e-6 + KL_LOSS_COEF=0.001 + ENTROPY_COEFF=0 + ROLLOUT_TP=1 + ROLLOUT_GPU_MEM_UTIL=0.7 + ROLLOUT_N=8 + PROJECT_NAME=verl_char_count + EXPERIMENT_NAME=char-count-smol135m-instruct-grpo-direct-1gpu + SAVE_FREQ=-1 + TEST_FREQ=5 + TOTAL_EPOCHS=5 + case "${DEVICE}" in + DATA=(algorithm.adv_estimator=grpo data.train_files=${TRAIN_FILE} data.val_files=${TEST_FILE} data.train_batch_size=${TRAIN_BATCH_SIZE} data.max_prompt_length=${MAX_PROMPT_LENGTH} data.max_response_length=${MAX_RESPONSE_LENGTH} data.filter_overlong_prompts=True data.truncation='error' algorithm.use_kl_in_reward=False) + MODEL=(actor_rollout_ref.model.path=${MODEL_PATH} actor_rollout_ref.model.use_remove_padding=True actor_rollout_ref.model.enable_gradient_checkpointing=True) + ACTOR=(actor_rollout_ref.actor.optim.lr=${ACTOR_LR} actor_rollout_ref.actor.ppo_mini_batch_size=${PPO_MINI_BATCH_SIZE} actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${PPO_MICRO_BATCH_SIZE_PER_GPU} actor_rollout_ref.actor.use_kl_loss=True actor_rollout_ref.actor.kl_loss_coef=${KL_LOSS_COEF} actor_rollout_ref.actor.kl_loss_type=low_var_kl actor_rollout_ref.actor.entropy_coeff=${ENTROPY_COEFF} actor_rollout_ref.actor.fsdp_config.param_offload=False actor_rollout_ref.actor.fsdp_config.optimizer_offload=False actor_rollout_ref.actor.ppo_max_token_len_per_gpu=3000 actor_rollout_ref.actor.use_dynamic_bsz=True) + ROLLOUT=(actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${LOG_PROB_MICRO_BATCH_SIZE_PER_GPU} actor_rollout_ref.rollout.tensor_model_parallel_size=${ROLLOUT_TP} actor_rollout_ref.rollout.name=${INFER_BACKEND} actor_rollout_ref.rollout.gpu_memory_utilization=${ROLLOUT_GPU_MEM_UTIL} actor_rollout_ref.rollout.enable_chunked_prefill=False actor_rollout_ref.rollout.enforce_eager=False actor_rollout_ref.rollout.free_cache_engine=True actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=4096 actor_rollout_ref.rollout.checkpoint_engine.update_weights_bucket_megabytes=4096 actor_rollout_ref.rollout.n=${ROLLOUT_N}) + REF=(actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${LOG_PROB_MICRO_BATCH_SIZE_PER_GPU} actor_rollout_ref.ref.fsdp_config.param_offload=True actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192) + TRAINER=(trainer.critic_warmup=0 trainer.logger='["console","wandb"]' trainer.project_name=${PROJECT_NAME} trainer.experiment_name=${EXPERIMENT_NAME} trainer.n_gpus_per_node=${NGPUS_PER_NODE} trainer.nnodes=${NNODES} trainer.save_freq=${SAVE_FREQ} trainer.test_freq=${TEST_FREQ} trainer.total_epochs=${TOTAL_EPOCHS}) + EXTRA=() + LAUNCH=(python3) + RAY=(ray_kwargs.ray_init.runtime_env.py_executable=null) + '[' 1 '!=' 0 ']' + '[' gpu = gpu ']' + '[' vllm = vllm ']' + LAUNCH=(uv run --frozen --all-packages --extra "${INFER_BACKEND}" --extra fsdp python3) + RAY=(ray_kwargs.ray_init.runtime_env.py_executable="uv -v run --frozen --all-packages --extra ${INFER_BACKEND} --extra fsdp") + uv run --frozen --all-packages --extra vllm --extra fsdp python3 -m verl.trainer.main_ppo algorithm.adv_estimator=grpo data.train_files=/data/char_count/rl/train.parquet data.val_files=/data/char_count/rl/test.parquet data.train_batch_size=128 data.max_prompt_length=128 data.max_response_length=128 data.filter_overlong_prompts=True data.truncation=error algorithm.use_kl_in_reward=False actor_rollout_ref.model.path=/models/SmolLM2-135M-Instruct actor_rollout_ref.model.use_remove_padding=True actor_rollout_ref.model.enable_gradient_checkpointing=True actor_rollout_ref.actor.optim.lr=1e-6 actor_rollout_ref.actor.ppo_mini_batch_size=16 actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 actor_rollout_ref.actor.use_kl_loss=True actor_rollout_ref.actor.kl_loss_coef=0.001 actor_rollout_ref.actor.kl_loss_type=low_var_kl actor_rollout_ref.actor.entropy_coeff=0 actor_rollout_ref.actor.fsdp_config.param_offload=False actor_rollout_ref.actor.fsdp_config.optimizer_offload=False actor_rollout_ref.actor.ppo_max_token_len_per_gpu=3000 actor_rollout_ref.actor.use_dynamic_bsz=True actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 actor_rollout_ref.rollout.tensor_model_parallel_size=1 actor_rollout_ref.rollout.name=vllm actor_rollout_ref.rollout.gpu_memory_utilization=0.7 actor_rollout_ref.rollout.enable_chunked_prefill=False actor_rollout_ref.rollout.enforce_eager=False actor_rollout_ref.rollout.free_cache_engine=True actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=4096 actor_rollout_ref.rollout.checkpoint_engine.update_weights_bucket_megabytes=4096 actor_rollout_ref.rollout.n=8 actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 actor_rollout_ref.ref.fsdp_config.param_offload=True actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192 trainer.critic_warmup=0 'trainer.logger=["console","wandb"]' trainer.project_name=verl_char_count trainer.experiment_name=char-count-smol135m-instruct-grpo-direct-1gpu trainer.n_gpus_per_node=1 trainer.nnodes=1 trainer.save_freq=-1 trainer.test_freq=5 trainer.total_epochs=5 'ray_kwargs.ray_init.runtime_env.py_executable=uv -v run --frozen --all-packages --extra vllm --extra fsdp' actor_rollout_ref.actor.strategy=fsdp2 actor_rollout_ref.actor.fsdp_config.param_offload=True actor_rollout_ref.actor.fsdp_config.optimizer_offload=True actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 actor_rollout_ref.actor.use_kl_loss=False actor_rollout_ref.actor.kl_loss_coef=0.0 actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8192 actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=8192 actor_rollout_ref.rollout.enforce_eager=True reward.custom_reward_function.path=/run/reward_tolerant.py reward.custom_reward_function.name=compute_score ++trainer.val_before_train=True 'trainer.logger=["console","mlflow"]' trainer.default_local_dir=/checkpoints/char-count-smol135m /workspace/verl/verl/trainer/main_ppo.py:181: UserWarning: Disabled critic as algorithm.adv_estimator != gae. If it is not intended, please set critic.enable=True use_critic=need_critic(config), 2026-11-01 16:48:39,246 INFO worker.py:2003 -- Started a local Ray instance. View the dashboard at http://:8265 /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/worker.py:2051: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0 warnings.warn( (raylet) DEBUG Found workspace root: `/workspace/verl` (raylet) DEBUG Adding root workspace member: `/workspace/verl` (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) (raylet) DEBUG Found project root: `/workspace/verl` (raylet) DEBUG No workspace root found, using project root (raylet) DEBUG Discovered project `verl` at: /workspace/verl (raylet) DEBUG No Python version file found in workspace: /workspace/verl (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata (raylet) DEBUG Checking for Python environment at: `.venv` (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` (raylet) DEBUG Skipping environment synchronization due to `--no-sync` (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=37761 --object-store-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:37513 --session-name=session_2026-11-01_16-48-32_811455_191 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=fa5e1a1d203a8a2c8e2dae1743d563c258a9a9fd2d3087d9f548d221 --worker-id=eb72bbf4a738f6042b29134d9cdfe95ec7cf239e55c6766d8040e91f --worker-launch-time-ms=1793551719790 --node-id=e3087f4a64dbc01fd839260bc555975b183b534e612ab28707facf9a --runtime-env-hash=247405026` (raylet) DEBUG Spawned child 2350 in process group 1 (TaskRunnerV1 pid=2350) WARNING:2026-11-01 16:48:58,097:Platform 'nvidia' (PlatformCUDA) is registered but not available. This may be due to this ray actor being a CPU-only actor. (TaskRunnerV1 pid=2350) /workspace/verl/verl/workers/engine/mindspeed/transformer_impl.py:24: UserWarning: NPU not support router replay for now. (TaskRunnerV1 pid=2350) from verl.utils.megatron.router_replay_patch import RouterReplay [validate_config] All configuration checks passed successfully! ray init kwargs: {'num_cpus': None, 'runtime_env': {'env_vars': {'TOKENIZERS_PARALLELISM': 'true', 'NCCL_DEBUG': 'WARN', 'VLLM_LOGGING_LEVEL': 'WARN', 'VLLM_ALLOW_RUNTIME_LORA_UPDATING': 'true', 'VLLM_DISABLE_COMPILE_CACHE': '1', 'HCCL_HOST_SOCKET_PORT_RANGE': 'auto', 'HCCL_NPU_SOCKET_PORT_RANGE': 'auto', 'HSA_NO_SCRATCH_RECLAIM': '1', 'PYTHONHASHSEED': '0', 'VERL_FULL_DETERMINISM': '0', 'VLLM_BATCH_INVARIANT': '0', 'VERL_RL_INSIGHT_ENABLE': '0'}, 'py_executable': 'uv -v run --frozen --all-packages --extra vllm --extra fsdp'}} (TaskRunnerV1 pid=2350) {'actor_rollout_ref': {'actor': {'_target_': 'verl.workers.config.FSDPActorConfig', (TaskRunnerV1 pid=2350) 'calculate_entropy': False, (TaskRunnerV1 pid=2350) 'calculate_sum_pi_squared': False, (TaskRunnerV1 pid=2350) 'checkpoint': {'_target_': 'verl.trainer.config.CheckpointConfig', (TaskRunnerV1 pid=2350) 'async_save': False, (TaskRunnerV1 pid=2350) 'load_contents': ['model', (TaskRunnerV1 pid=2350) 'optimizer', (TaskRunnerV1 pid=2350) 'extra'], (TaskRunnerV1 pid=2350) 'save_contents': ['model', (TaskRunnerV1 pid=2350) 'optimizer', (TaskRunnerV1 pid=2350) 'extra'], (TaskRunnerV1 pid=2350) 'strict': True}, (TaskRunnerV1 pid=2350) 'clip_ratio': 0.2, (TaskRunnerV1 pid=2350) 'clip_ratio_c': 3.0, (TaskRunnerV1 pid=2350) 'clip_ratio_high': 0.2, (TaskRunnerV1 pid=2350) 'clip_ratio_low': 0.2, (TaskRunnerV1 pid=2350) 'data_loader_seed': 42, (TaskRunnerV1 pid=2350) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2350) 'entropy_coeff': 0, (TaskRunnerV1 pid=2350) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2350) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2350) 'freeze_vision_tower': False, (TaskRunnerV1 pid=2350) 'fsdp_config': {'_target_': 'verl.workers.config.FSDPEngineConfig', (TaskRunnerV1 pid=2350) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2350) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2350) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2350) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2350) 'forward_only': False, (TaskRunnerV1 pid=2350) 'forward_prefetch': False, (TaskRunnerV1 pid=2350) 'fsdp_size': -1, (TaskRunnerV1 pid=2350) 'full_determinism': False, (TaskRunnerV1 pid=2350) 'model_dtype': 'bfloat16', (TaskRunnerV1 pid=2350) 'offload_policy': False, (TaskRunnerV1 pid=2350) 'optimizer_offload': True, (TaskRunnerV1 pid=2350) 'param_offload': True, (TaskRunnerV1 pid=2350) 'qat': {'_target_': 'verl.workers.config.QATEngineConfig', (TaskRunnerV1 pid=2350) 'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'group_size': 16, (TaskRunnerV1 pid=2350) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2350) 'embed_tokens', (TaskRunnerV1 pid=2350) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2350) 'mode': 'w4a16', (TaskRunnerV1 pid=2350) 'quantization_config_path': None}, (TaskRunnerV1 pid=2350) 'reshard_after_forward': True, (TaskRunnerV1 pid=2350) 'seed': 42, (TaskRunnerV1 pid=2350) 'strategy': 'fsdp', (TaskRunnerV1 pid=2350) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2350) 'use_orig_params': False, (TaskRunnerV1 pid=2350) 'use_torch_compile': True, (TaskRunnerV1 pid=2350) 'wrap_policy': {'min_num_params': 0}}, (TaskRunnerV1 pid=2350) 'grad_clip': 1.0, (TaskRunnerV1 pid=2350) 'kl_loss_coef': 0.0, (TaskRunnerV1 pid=2350) 'kl_loss_type': 'low_var_kl', (TaskRunnerV1 pid=2350) 'loss_agg_mode': 'token-mean', (TaskRunnerV1 pid=2350) 'loss_scale_factor': None, (TaskRunnerV1 pid=2350) 'optim': {'_target_': 'verl.workers.config.FSDPOptimizerConfig', (TaskRunnerV1 pid=2350) 'betas': [0.9, 0.999], (TaskRunnerV1 pid=2350) 'clip_grad': 1.0, (TaskRunnerV1 pid=2350) 'lr': 1e-06, (TaskRunnerV1 pid=2350) 'lr_scheduler_type': 'constant', (TaskRunnerV1 pid=2350) 'lr_warmup_steps': -1, (TaskRunnerV1 pid=2350) 'lr_warmup_steps_ratio': 0.0, (TaskRunnerV1 pid=2350) 'min_lr_ratio': 0.0, (TaskRunnerV1 pid=2350) 'num_cycles': 0.5, (TaskRunnerV1 pid=2350) 'optimizer': 'AdamW', (TaskRunnerV1 pid=2350) 'optimizer_impl': 'torch.optim', (TaskRunnerV1 pid=2350) 'override_optimizer_config': None, (TaskRunnerV1 pid=2350) 'total_training_steps': -1, (TaskRunnerV1 pid=2350) 'warmup_style': None, (TaskRunnerV1 pid=2350) 'weight_decay': 0.01, (TaskRunnerV1 pid=2350) 'zero_indexed_step': True}, (TaskRunnerV1 pid=2350) 'policy_loss': {'_target_': 'verl.workers.config.PolicyLossConfig', (TaskRunnerV1 pid=2350) 'clip_cov_lb': 1.0, (TaskRunnerV1 pid=2350) 'clip_cov_ratio': 0.0002, (TaskRunnerV1 pid=2350) 'clip_cov_ub': 5.0, (TaskRunnerV1 pid=2350) 'kl_cov_ratio': 0.0002, (TaskRunnerV1 pid=2350) 'loss_mode': 'vanilla', (TaskRunnerV1 pid=2350) 'ppo_kl_coef': 0.1}, (TaskRunnerV1 pid=2350) 'ppo_epochs': 1, (TaskRunnerV1 pid=2350) 'ppo_max_token_len_per_gpu': 8192, (TaskRunnerV1 pid=2350) 'ppo_micro_batch_size': None, (TaskRunnerV1 pid=2350) 'ppo_micro_batch_size_per_gpu': 8, (TaskRunnerV1 pid=2350) 'ppo_mini_batch_size': 16, (TaskRunnerV1 pid=2350) 'profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2350) 'all_ranks': False, (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'ranks': [], (TaskRunnerV1 pid=2350) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2350) 'tool': None, (TaskRunnerV1 pid=2350) 'tool_config': {'npu': {'_target_': 'verl.utils.profiler.config.NPUToolConfig', (TaskRunnerV1 pid=2350) 'analysis': True, (TaskRunnerV1 pid=2350) 'contents': [], (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'level': 'level0'}, (TaskRunnerV1 pid=2350) 'nsys': {'_target_': 'verl.utils.profiler.config.NsightToolConfig', (TaskRunnerV1 pid=2350) 'discrete': False}, (TaskRunnerV1 pid=2350) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2350) 'config_path': None, (TaskRunnerV1 pid=2350) 'stages': None, (TaskRunnerV1 pid=2350) 'steps': None, (TaskRunnerV1 pid=2350) 'strict': False}, (TaskRunnerV1 pid=2350) 'torch': {'_target_': 'verl.utils.profiler.config.TorchProfilerToolConfig', (TaskRunnerV1 pid=2350) 'contents': [], (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'schedule': {'_target_': 'verl.utils.profiler.config.TorchProfilerScheduleConfig', (TaskRunnerV1 pid=2350) 'active': 0, (TaskRunnerV1 pid=2350) 'repeat': 0, (TaskRunnerV1 pid=2350) 'skip_first': 0, (TaskRunnerV1 pid=2350) 'wait': 0, (TaskRunnerV1 pid=2350) 'warmup': 0}}, (TaskRunnerV1 pid=2350) 'torch_memory': {'_target_': 'verl.utils.profiler.config.TorchMemoryToolConfig', (TaskRunnerV1 pid=2350) 'stack_depth': 32, (TaskRunnerV1 pid=2350) 'trace_alloc_max_entries': 100000}}}, (TaskRunnerV1 pid=2350) 'qat': {'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'group_size': 16, (TaskRunnerV1 pid=2350) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2350) 'embed_tokens', (TaskRunnerV1 pid=2350) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2350) 'mode': 'w4a16', (TaskRunnerV1 pid=2350) 'quantization_config_path': None}, (TaskRunnerV1 pid=2350) 'rollout_n': 8, (TaskRunnerV1 pid=2350) 'router_replay': {'_target_': 'verl.workers.config.RouterReplayConfig', (TaskRunnerV1 pid=2350) 'mode': 'disabled', (TaskRunnerV1 pid=2350) 'record_file': None, (TaskRunnerV1 pid=2350) 'replay_file': None}, (TaskRunnerV1 pid=2350) 'shuffle': False, (TaskRunnerV1 pid=2350) 'strategy': 'fsdp2', (TaskRunnerV1 pid=2350) 'tau_neg': 1.05, (TaskRunnerV1 pid=2350) 'tau_pos': 1.0, (TaskRunnerV1 pid=2350) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2350) 'use_dynamic_bsz': True, (TaskRunnerV1 pid=2350) 'use_fused_kernels': False, (TaskRunnerV1 pid=2350) 'use_kl_loss': False, (TaskRunnerV1 pid=2350) 'use_prefix_grouper': False, (TaskRunnerV1 pid=2350) 'use_remove_padding': True, (TaskRunnerV1 pid=2350) 'use_torch_compile': True}, (TaskRunnerV1 pid=2350) 'hybrid_engine': True, (TaskRunnerV1 pid=2350) 'model': {'_target_': 'verl.workers.config.HFModelConfig', (TaskRunnerV1 pid=2350) 'custom_chat_template': None, (TaskRunnerV1 pid=2350) 'enable_activation_offload': False, (TaskRunnerV1 pid=2350) 'enable_gradient_checkpointing': True, (TaskRunnerV1 pid=2350) 'exclude_modules': None, (TaskRunnerV1 pid=2350) 'external_lib': None, (TaskRunnerV1 pid=2350) 'fused_kernel_options': {'impl_backend': 'torch'}, (TaskRunnerV1 pid=2350) 'hf_config_path': None, (TaskRunnerV1 pid=2350) 'lora': {'a2a_experimental': False, (TaskRunnerV1 pid=2350) 'adapter_path': None, (TaskRunnerV1 pid=2350) 'alpha': 32, (TaskRunnerV1 pid=2350) 'dropout': 0.0, (TaskRunnerV1 pid=2350) 'dropout_position': 'pre', (TaskRunnerV1 pid=2350) 'dtype': None, (TaskRunnerV1 pid=2350) 'exclude_modules': [], (TaskRunnerV1 pid=2350) 'freeze_language_model': True, (TaskRunnerV1 pid=2350) 'freeze_vision_model': True, (TaskRunnerV1 pid=2350) 'freeze_vision_projection': True, (TaskRunnerV1 pid=2350) 'lora_A_init_method': 'xavier', (TaskRunnerV1 pid=2350) 'lora_B_init_method': 'zero', (TaskRunnerV1 pid=2350) 'merge': False, (TaskRunnerV1 pid=2350) 'rank': 0, (TaskRunnerV1 pid=2350) 'target_modules': ['linear_qkv', (TaskRunnerV1 pid=2350) 'linear_proj', (TaskRunnerV1 pid=2350) 'linear_fc1', (TaskRunnerV1 pid=2350) 'linear_fc2'], (TaskRunnerV1 pid=2350) 'type': 'lora'}, (TaskRunnerV1 pid=2350) 'lora_adapter_path': None, (TaskRunnerV1 pid=2350) 'lora_alpha': 16, (TaskRunnerV1 pid=2350) 'lora_rank': 0, (TaskRunnerV1 pid=2350) 'mtp': {'_target_': 'verl.workers.config.MtpConfig', (TaskRunnerV1 pid=2350) 'detach_encoder': False, (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'enable_rollout': False, (TaskRunnerV1 pid=2350) 'enable_train': False, (TaskRunnerV1 pid=2350) 'method': 'mtp', (TaskRunnerV1 pid=2350) 'mtp_loss_scaling_factor': 0.1, (TaskRunnerV1 pid=2350) 'num_speculative_tokens': 1, (TaskRunnerV1 pid=2350) 'speculative_algorithm': 'EAGLE', (TaskRunnerV1 pid=2350) 'speculative_eagle_topk': 1, (TaskRunnerV1 pid=2350) 'speculative_num_draft_tokens': 4, (TaskRunnerV1 pid=2350) 'speculative_num_steps': 3}, (TaskRunnerV1 pid=2350) 'override_config': {}, (TaskRunnerV1 pid=2350) 'path': '/models/SmolLM2-135M-Instruct', (TaskRunnerV1 pid=2350) 'target_modules': 'all-linear', (TaskRunnerV1 pid=2350) 'tiled_mlp': {'enabled': False, (TaskRunnerV1 pid=2350) 'num_shards': 4}, (TaskRunnerV1 pid=2350) 'tokenizer_path': None, (TaskRunnerV1 pid=2350) 'trust_remote_code': False, (TaskRunnerV1 pid=2350) 'use_fused_kernels': False, (TaskRunnerV1 pid=2350) 'use_liger': False, (TaskRunnerV1 pid=2350) 'use_remove_padding': True, (TaskRunnerV1 pid=2350) 'use_shm': False}, (TaskRunnerV1 pid=2350) 'nccl_timeout': 600, (TaskRunnerV1 pid=2350) 'ref': {'_target_': 'verl.workers.config.FSDPActorConfig', (TaskRunnerV1 pid=2350) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2350) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2350) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2350) 'fsdp_config': {'_target_': 'verl.workers.config.FSDPEngineConfig', (TaskRunnerV1 pid=2350) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2350) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2350) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2350) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2350) 'forward_only': True, (TaskRunnerV1 pid=2350) 'forward_prefetch': False, (TaskRunnerV1 pid=2350) 'fsdp_size': -1, (TaskRunnerV1 pid=2350) 'full_determinism': False, (TaskRunnerV1 pid=2350) 'model_dtype': 'fp32', (TaskRunnerV1 pid=2350) 'offload_policy': False, (TaskRunnerV1 pid=2350) 'optimizer_offload': False, (TaskRunnerV1 pid=2350) 'param_offload': True, (TaskRunnerV1 pid=2350) 'qat': {'_target_': 'verl.workers.config.QATEngineConfig', (TaskRunnerV1 pid=2350) 'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'group_size': 16, (TaskRunnerV1 pid=2350) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2350) 'embed_tokens', (TaskRunnerV1 pid=2350) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2350) 'mode': 'w4a16', (TaskRunnerV1 pid=2350) 'quantization_config_path': None}, (TaskRunnerV1 pid=2350) 'reshard_after_forward': True, (TaskRunnerV1 pid=2350) 'seed': 42, (TaskRunnerV1 pid=2350) 'strategy': 'fsdp', (TaskRunnerV1 pid=2350) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2350) 'use_orig_params': False, (TaskRunnerV1 pid=2350) 'use_torch_compile': True, (TaskRunnerV1 pid=2350) 'wrap_policy': {'min_num_params': 0}}, (TaskRunnerV1 pid=2350) 'log_prob_max_token_len_per_gpu': 8192, (TaskRunnerV1 pid=2350) 'log_prob_micro_batch_size': None, (TaskRunnerV1 pid=2350) 'log_prob_micro_batch_size_per_gpu': 8, (TaskRunnerV1 pid=2350) 'log_prob_use_dynamic_bsz': True, (TaskRunnerV1 pid=2350) 'profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2350) 'all_ranks': False, (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'ranks': [], (TaskRunnerV1 pid=2350) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2350) 'tool': None, (TaskRunnerV1 pid=2350) 'tool_config': {'npu': {'_target_': 'verl.utils.profiler.config.NPUToolConfig', (TaskRunnerV1 pid=2350) 'analysis': True, (TaskRunnerV1 pid=2350) 'contents': [], (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'level': 'level0'}, (TaskRunnerV1 pid=2350) 'nsys': {'_target_': 'verl.utils.profiler.config.NsightToolConfig', (TaskRunnerV1 pid=2350) 'discrete': False}, (TaskRunnerV1 pid=2350) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2350) 'config_path': None, (TaskRunnerV1 pid=2350) 'stages': None, (TaskRunnerV1 pid=2350) 'steps': None, (TaskRunnerV1 pid=2350) 'strict': False}, (TaskRunnerV1 pid=2350) 'torch': {'_target_': 'verl.utils.profiler.config.TorchProfilerToolConfig', (TaskRunnerV1 pid=2350) 'contents': [], (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'schedule': {'_target_': 'verl.utils.profiler.config.TorchProfilerScheduleConfig', (TaskRunnerV1 pid=2350) 'active': 0, (TaskRunnerV1 pid=2350) 'repeat': 0, (TaskRunnerV1 pid=2350) 'skip_first': 0, (TaskRunnerV1 pid=2350) 'wait': 0, (TaskRunnerV1 pid=2350) 'warmup': 0}}, (TaskRunnerV1 pid=2350) 'torch_memory': {'_target_': 'verl.utils.profiler.config.TorchMemoryToolConfig', (TaskRunnerV1 pid=2350) 'stack_depth': 32, (TaskRunnerV1 pid=2350) 'trace_alloc_max_entries': 100000}}}, (TaskRunnerV1 pid=2350) 'rollout_n': 8, (TaskRunnerV1 pid=2350) 'router_replay': {'_target_': 'verl.workers.config.RouterReplayConfig', (TaskRunnerV1 pid=2350) 'mode': 'disabled', (TaskRunnerV1 pid=2350) 'record_file': None, (TaskRunnerV1 pid=2350) 'replay_file': None}, (TaskRunnerV1 pid=2350) 'strategy': 'fsdp2', (TaskRunnerV1 pid=2350) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2350) 'use_torch_compile': True}, (TaskRunnerV1 pid=2350) 'rollout': {'_target_': 'verl.workers.config.RolloutConfig', (TaskRunnerV1 pid=2350) 'agent': {'_target_': 'verl.workers.config.AgentLoopConfig', (TaskRunnerV1 pid=2350) 'agent_loop_config_path': None, (TaskRunnerV1 pid=2350) 'custom_async_server': {'_target_': 'verl.workers.config.CustomAsyncServerConfig', (TaskRunnerV1 pid=2350) 'name': None, (TaskRunnerV1 pid=2350) 'path': None}, (TaskRunnerV1 pid=2350) 'default_agent_loop': 'single_turn_agent', (TaskRunnerV1 pid=2350) 'num_workers': 8}, (TaskRunnerV1 pid=2350) 'calculate_log_probs': True, (TaskRunnerV1 pid=2350) 'checkpoint_engine': {'_target_': 'verl.workers.config.CheckpointEngineConfig', (TaskRunnerV1 pid=2350) 'backend': 'naive', (TaskRunnerV1 pid=2350) 'custom_backend_module': None, (TaskRunnerV1 pid=2350) 'engine_kwargs': {}, (TaskRunnerV1 pid=2350) 'update_weights_bucket_megabytes': 4096}, (TaskRunnerV1 pid=2350) 'cudagraph_capture_sizes': None, (TaskRunnerV1 pid=2350) 'data_parallel_size': 1, (TaskRunnerV1 pid=2350) 'disable_log_stats': True, (TaskRunnerV1 pid=2350) 'disaggregation': {'bootstrap_port': None, (TaskRunnerV1 pid=2350) 'decode_replicas': 1, (TaskRunnerV1 pid=2350) 'decode_tensor_model_parallel_size': None, (TaskRunnerV1 pid=2350) 'enabled': False, (TaskRunnerV1 pid=2350) 'ib_device': None, (TaskRunnerV1 pid=2350) 'prefill_replicas': 1, (TaskRunnerV1 pid=2350) 'transfer_backend': 'nixl'}, (TaskRunnerV1 pid=2350) 'do_sample': True, (TaskRunnerV1 pid=2350) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2350) 'enable_chunked_prefill': False, (TaskRunnerV1 pid=2350) 'enable_prefix_caching': True, (TaskRunnerV1 pid=2350) 'enable_rollout_routing_replay': False, (TaskRunnerV1 pid=2350) 'enforce_eager': True, (TaskRunnerV1 pid=2350) 'engine_kwargs': {'sglang': {}, (TaskRunnerV1 pid=2350) 'trtllm': {}, (TaskRunnerV1 pid=2350) 'vllm': {}}, (TaskRunnerV1 pid=2350) 'expert_parallel_size': 1, (TaskRunnerV1 pid=2350) 'free_cache_engine': True, (TaskRunnerV1 pid=2350) 'full_determinism': False, (TaskRunnerV1 pid=2350) 'gpu_memory_utilization': 0.7, (TaskRunnerV1 pid=2350) 'ignore_eos': False, (TaskRunnerV1 pid=2350) 'layered_summon': False, (TaskRunnerV1 pid=2350) 'load_format': 'dummy', (TaskRunnerV1 pid=2350) 'log_prob_max_token_len_per_gpu': 8192, (TaskRunnerV1 pid=2350) 'log_prob_micro_batch_size': None, (TaskRunnerV1 pid=2350) 'log_prob_micro_batch_size_per_gpu': 8, (TaskRunnerV1 pid=2350) 'log_prob_use_dynamic_bsz': True, (TaskRunnerV1 pid=2350) 'logprobs_mode': 'processed_logprobs', (TaskRunnerV1 pid=2350) 'max_model_len': None, (TaskRunnerV1 pid=2350) 'max_num_batched_tokens': 8192, (TaskRunnerV1 pid=2350) 'max_num_seqs': 1024, (TaskRunnerV1 pid=2350) 'mode': 'async', (TaskRunnerV1 pid=2350) 'moe_load_balance_metrics_interval': 0, (TaskRunnerV1 pid=2350) 'mtp': {'_target_': 'verl.workers.config.MtpConfig', (TaskRunnerV1 pid=2350) 'detach_encoder': False, (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'enable_rollout': False, (TaskRunnerV1 pid=2350) 'enable_train': False, (TaskRunnerV1 pid=2350) 'method': 'mtp', (TaskRunnerV1 pid=2350) 'mtp_loss_scaling_factor': 0.1, (TaskRunnerV1 pid=2350) 'num_speculative_tokens': 1, (TaskRunnerV1 pid=2350) 'speculative_algorithm': 'EAGLE', (TaskRunnerV1 pid=2350) 'speculative_eagle_topk': 1, (TaskRunnerV1 pid=2350) 'speculative_num_draft_tokens': 4, (TaskRunnerV1 pid=2350) 'speculative_num_steps': 3}, (TaskRunnerV1 pid=2350) 'multi_stage_wake_up': False, (TaskRunnerV1 pid=2350) 'multi_turn': {'_target_': 'verl.workers.config.MultiTurnConfig', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'format': 'hermes', (TaskRunnerV1 pid=2350) 'function_tool_path': None, (TaskRunnerV1 pid=2350) 'max_assistant_turns': None, (TaskRunnerV1 pid=2350) 'max_parallel_calls': 1, (TaskRunnerV1 pid=2350) 'max_tool_response_length': 256, (TaskRunnerV1 pid=2350) 'max_user_turns': None, (TaskRunnerV1 pid=2350) 'num_repeat_rollouts': None, (TaskRunnerV1 pid=2350) 'tokenization_sanity_check_mode': 'strict', (TaskRunnerV1 pid=2350) 'tool_config_path': None, (TaskRunnerV1 pid=2350) 'tool_response_truncate_side': 'middle', (TaskRunnerV1 pid=2350) 'use_inference_chat_template': False}, (TaskRunnerV1 pid=2350) 'n': 8, (TaskRunnerV1 pid=2350) 'n_gpus_per_node': 1, (TaskRunnerV1 pid=2350) 'name': 'vllm', (TaskRunnerV1 pid=2350) 'nnodes': 0, (TaskRunnerV1 pid=2350) 'over_sample_rate': 0, (TaskRunnerV1 pid=2350) 'pipeline_model_parallel_size': 1, (TaskRunnerV1 pid=2350) 'profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2350) 'all_ranks': False, (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'ranks': [], (TaskRunnerV1 pid=2350) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2350) 'tool': None, (TaskRunnerV1 pid=2350) 'tool_config': {'npu': {'_target_': 'verl.utils.profiler.config.NPUToolConfig', (TaskRunnerV1 pid=2350) 'analysis': True, (TaskRunnerV1 pid=2350) 'contents': [], (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'level': 'level0', (TaskRunnerV1 pid=2350) 'profile_token_end': None, (TaskRunnerV1 pid=2350) 'profile_token_start': None}, (TaskRunnerV1 pid=2350) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2350) 'config_path': None, (TaskRunnerV1 pid=2350) 'stages': None, (TaskRunnerV1 pid=2350) 'steps': None, (TaskRunnerV1 pid=2350) 'strict': False}, (TaskRunnerV1 pid=2350) 'torch': {'_target_': 'verl.utils.profiler.config.TorchProfilerToolConfig', (TaskRunnerV1 pid=2350) 'contents': [], (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'profile_token_end': None, (TaskRunnerV1 pid=2350) 'profile_token_start': None}}}, (TaskRunnerV1 pid=2350) 'prometheus': {'_target_': 'verl.workers.config.PrometheusConfig', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'file': '/tmp/ray/session_latest/metrics/prometheus/prometheus.yml', (TaskRunnerV1 pid=2350) 'port': 9090, (TaskRunnerV1 pid=2350) 'served_model_name': '/models/SmolLM2-135M-Instruct'}, (TaskRunnerV1 pid=2350) 'prompt_length': 128, (TaskRunnerV1 pid=2350) 'qat': {'_target_': 'verl.workers.config.QATEngineConfig', (TaskRunnerV1 pid=2350) 'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'group_size': 16, (TaskRunnerV1 pid=2350) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2350) 'embed_tokens', (TaskRunnerV1 pid=2350) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2350) 'mode': 'w4a16', (TaskRunnerV1 pid=2350) 'quantization_config_path': None}, (TaskRunnerV1 pid=2350) 'quantization': None, (TaskRunnerV1 pid=2350) 'quantization_config_file': None, (TaskRunnerV1 pid=2350) 'response_length': 128, (TaskRunnerV1 pid=2350) 'scheduling_policy': 'fcfs', (TaskRunnerV1 pid=2350) 'seed': 42, (TaskRunnerV1 pid=2350) 'skip_tokenizer_init': True, (TaskRunnerV1 pid=2350) 'temperature': 1.0, (TaskRunnerV1 pid=2350) 'tensor_model_parallel_size': 1, (TaskRunnerV1 pid=2350) 'top_k': -1, (TaskRunnerV1 pid=2350) 'top_p': 1, (TaskRunnerV1 pid=2350) 'trace': {'_target_': 'verl.workers.config.TraceConfig', (TaskRunnerV1 pid=2350) 'backend': None, (TaskRunnerV1 pid=2350) 'experiment_name': 'char-count-smol135m-instruct-grpo-direct-1gpu', (TaskRunnerV1 pid=2350) 'max_samples_per_step_per_worker': None, (TaskRunnerV1 pid=2350) 'project_name': 'verl_char_count', (TaskRunnerV1 pid=2350) 'token2text': False}, (TaskRunnerV1 pid=2350) 'val_kwargs': {'_target_': 'verl.workers.config.SamplingConfig', (TaskRunnerV1 pid=2350) 'do_sample': False, (TaskRunnerV1 pid=2350) 'n': 1, (TaskRunnerV1 pid=2350) 'temperature': 0, (TaskRunnerV1 pid=2350) 'top_k': -1, (TaskRunnerV1 pid=2350) 'top_p': 1.0}}}, (TaskRunnerV1 pid=2350) 'algorithm': {'_target_': 'verl.trainer.config.AlgoConfig', (TaskRunnerV1 pid=2350) 'adv_estimator': 'grpo', (TaskRunnerV1 pid=2350) 'filter_groups': {'_target_': 'verl.trainer.config.FilterGroupsConfig', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'max_inflight_gen_batches': 1, (TaskRunnerV1 pid=2350) 'max_num_gen_batches': 0, (TaskRunnerV1 pid=2350) 'metric': None}, (TaskRunnerV1 pid=2350) 'gamma': 1.0, (TaskRunnerV1 pid=2350) 'kl_ctrl': {'_target_': 'verl.trainer.config.KLControlConfig', (TaskRunnerV1 pid=2350) 'horizon': 10000, (TaskRunnerV1 pid=2350) 'kl_coef': 0.001, (TaskRunnerV1 pid=2350) 'target_kl': 0.1, (TaskRunnerV1 pid=2350) 'type': 'fixed'}, (TaskRunnerV1 pid=2350) 'kl_penalty': 'kl', (TaskRunnerV1 pid=2350) 'lam': 1.0, (TaskRunnerV1 pid=2350) 'norm_adv_by_std_in_grpo': True, (TaskRunnerV1 pid=2350) 'pf_ppo': {'reweight_method': 'pow', 'weight_pow': 2.0}, (TaskRunnerV1 pid=2350) 'rollout_correction': {'bypass_mode': False, (TaskRunnerV1 pid=2350) 'loss_type': 'ppo_clip', (TaskRunnerV1 pid=2350) 'rollout_is': None, (TaskRunnerV1 pid=2350) 'rollout_is_batch_normalize': False, (TaskRunnerV1 pid=2350) 'rollout_is_threshold': 2.0, (TaskRunnerV1 pid=2350) 'rollout_rs': None, (TaskRunnerV1 pid=2350) 'rollout_rs_threshold': None}, (TaskRunnerV1 pid=2350) 'use_kl_in_reward': False, (TaskRunnerV1 pid=2350) 'use_pf_ppo': False}, (TaskRunnerV1 pid=2350) 'critic': {'_target_': 'verl.workers.config.FSDPCriticConfig', (TaskRunnerV1 pid=2350) 'checkpoint': {'_target_': 'verl.trainer.config.CheckpointConfig', (TaskRunnerV1 pid=2350) 'async_save': False, (TaskRunnerV1 pid=2350) 'load_contents': ['model', 'optimizer', 'extra'], (TaskRunnerV1 pid=2350) 'save_contents': ['model', 'optimizer', 'extra']}, (TaskRunnerV1 pid=2350) 'cliprange_value': 0.5, (TaskRunnerV1 pid=2350) 'data_loader_seed': 42, (TaskRunnerV1 pid=2350) 'enable': None, (TaskRunnerV1 pid=2350) 'forward_max_token_len_per_gpu': 32768, (TaskRunnerV1 pid=2350) 'forward_micro_batch_size': None, (TaskRunnerV1 pid=2350) 'forward_micro_batch_size_per_gpu': None, (TaskRunnerV1 pid=2350) 'fsdp': {'_target_': 'verl.workers.config.FSDPEngineConfig', (TaskRunnerV1 pid=2350) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2350) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2350) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2350) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2350) 'forward_only': False, (TaskRunnerV1 pid=2350) 'forward_prefetch': False, (TaskRunnerV1 pid=2350) 'fsdp_size': -1, (TaskRunnerV1 pid=2350) 'full_determinism': False, (TaskRunnerV1 pid=2350) 'model_dtype': 'fp32', (TaskRunnerV1 pid=2350) 'offload_policy': False, (TaskRunnerV1 pid=2350) 'optimizer_offload': False, (TaskRunnerV1 pid=2350) 'param_offload': False, (TaskRunnerV1 pid=2350) 'qat': {'_target_': 'verl.workers.config.QATEngineConfig', (TaskRunnerV1 pid=2350) 'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'group_size': 16, (TaskRunnerV1 pid=2350) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2350) 'embed_tokens', (TaskRunnerV1 pid=2350) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2350) 'mode': 'w4a16', (TaskRunnerV1 pid=2350) 'quantization_config_path': None}, (TaskRunnerV1 pid=2350) 'reshard_after_forward': True, (TaskRunnerV1 pid=2350) 'seed': 42, (TaskRunnerV1 pid=2350) 'strategy': 'fsdp', (TaskRunnerV1 pid=2350) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2350) 'use_orig_params': False, (TaskRunnerV1 pid=2350) 'use_torch_compile': True, (TaskRunnerV1 pid=2350) 'wrap_policy': {'min_num_params': 0}}, (TaskRunnerV1 pid=2350) 'grad_clip': 1.0, (TaskRunnerV1 pid=2350) 'loss_agg_mode': 'token-mean', (TaskRunnerV1 pid=2350) 'loss_scale_factor': None, (TaskRunnerV1 pid=2350) 'model': {'_target_': 'verl.workers.config.HFModelConfig', (TaskRunnerV1 pid=2350) 'custom_chat_template': None, (TaskRunnerV1 pid=2350) 'enable_activation_offload': False, (TaskRunnerV1 pid=2350) 'enable_gradient_checkpointing': True, (TaskRunnerV1 pid=2350) 'exclude_modules': None, (TaskRunnerV1 pid=2350) 'external_lib': None, (TaskRunnerV1 pid=2350) 'fused_kernel_options': {'impl_backend': 'torch'}, (TaskRunnerV1 pid=2350) 'hf_config_path': None, (TaskRunnerV1 pid=2350) 'lora': {'a2a_experimental': False, (TaskRunnerV1 pid=2350) 'adapter_path': None, (TaskRunnerV1 pid=2350) 'alpha': 32, (TaskRunnerV1 pid=2350) 'dropout': 0.0, (TaskRunnerV1 pid=2350) 'dropout_position': 'pre', (TaskRunnerV1 pid=2350) 'dtype': None, (TaskRunnerV1 pid=2350) 'exclude_modules': [], (TaskRunnerV1 pid=2350) 'freeze_language_model': True, (TaskRunnerV1 pid=2350) 'freeze_vision_model': True, (TaskRunnerV1 pid=2350) 'freeze_vision_projection': True, (TaskRunnerV1 pid=2350) 'lora_A_init_method': 'xavier', (TaskRunnerV1 pid=2350) 'lora_B_init_method': 'zero', (TaskRunnerV1 pid=2350) 'merge': False, (TaskRunnerV1 pid=2350) 'rank': 0, (TaskRunnerV1 pid=2350) 'target_modules': ['linear_qkv', (TaskRunnerV1 pid=2350) 'linear_proj', (TaskRunnerV1 pid=2350) 'linear_fc1', (TaskRunnerV1 pid=2350) 'linear_fc2'], (TaskRunnerV1 pid=2350) 'type': 'lora'}, (TaskRunnerV1 pid=2350) 'lora_adapter_path': None, (TaskRunnerV1 pid=2350) 'lora_alpha': 16, (TaskRunnerV1 pid=2350) 'lora_rank': 0, (TaskRunnerV1 pid=2350) 'mtp': {'_target_': 'verl.workers.config.MtpConfig', (TaskRunnerV1 pid=2350) 'detach_encoder': False, (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'enable_rollout': False, (TaskRunnerV1 pid=2350) 'enable_train': False, (TaskRunnerV1 pid=2350) 'method': 'mtp', (TaskRunnerV1 pid=2350) 'mtp_loss_scaling_factor': 0.1, (TaskRunnerV1 pid=2350) 'num_speculative_tokens': 1, (TaskRunnerV1 pid=2350) 'speculative_algorithm': 'EAGLE', (TaskRunnerV1 pid=2350) 'speculative_eagle_topk': 1, (TaskRunnerV1 pid=2350) 'speculative_num_draft_tokens': 4, (TaskRunnerV1 pid=2350) 'speculative_num_steps': 3}, (TaskRunnerV1 pid=2350) 'override_config': {}, (TaskRunnerV1 pid=2350) 'path': '~/models/deepseek-llm-7b-chat', (TaskRunnerV1 pid=2350) 'target_modules': 'all-linear', (TaskRunnerV1 pid=2350) 'tiled_mlp': {'enabled': False, 'num_shards': 4}, (TaskRunnerV1 pid=2350) 'tokenizer_path': None, (TaskRunnerV1 pid=2350) 'trust_remote_code': False, (TaskRunnerV1 pid=2350) 'use_fused_kernels': False, (TaskRunnerV1 pid=2350) 'use_liger': False, (TaskRunnerV1 pid=2350) 'use_remove_padding': True, (TaskRunnerV1 pid=2350) 'use_shm': False}, (TaskRunnerV1 pid=2350) 'optim': {'_target_': 'verl.workers.config.FSDPOptimizerConfig', (TaskRunnerV1 pid=2350) 'betas': [0.9, 0.999], (TaskRunnerV1 pid=2350) 'clip_grad': 1.0, (TaskRunnerV1 pid=2350) 'lr': 1e-05, (TaskRunnerV1 pid=2350) 'lr_scheduler_type': 'constant', (TaskRunnerV1 pid=2350) 'lr_warmup_steps': -1, (TaskRunnerV1 pid=2350) 'lr_warmup_steps_ratio': 0.0, (TaskRunnerV1 pid=2350) 'min_lr_ratio': 0.0, (TaskRunnerV1 pid=2350) 'num_cycles': 0.5, (TaskRunnerV1 pid=2350) 'optimizer': 'AdamW', (TaskRunnerV1 pid=2350) 'optimizer_impl': 'torch.optim', (TaskRunnerV1 pid=2350) 'override_optimizer_config': None, (TaskRunnerV1 pid=2350) 'total_training_steps': -1, (TaskRunnerV1 pid=2350) 'warmup_style': None, (TaskRunnerV1 pid=2350) 'weight_decay': 0.01, (TaskRunnerV1 pid=2350) 'zero_indexed_step': True}, (TaskRunnerV1 pid=2350) 'ppo_epochs': 1, (TaskRunnerV1 pid=2350) 'ppo_max_token_len_per_gpu': 32768, (TaskRunnerV1 pid=2350) 'ppo_micro_batch_size': None, (TaskRunnerV1 pid=2350) 'ppo_micro_batch_size_per_gpu': None, (TaskRunnerV1 pid=2350) 'ppo_mini_batch_size': 16, (TaskRunnerV1 pid=2350) 'profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2350) 'all_ranks': False, (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'ranks': [], (TaskRunnerV1 pid=2350) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2350) 'tool': None, (TaskRunnerV1 pid=2350) 'tool_config': {'npu': {'_target_': 'verl.utils.profiler.config.NPUToolConfig', (TaskRunnerV1 pid=2350) 'analysis': True, (TaskRunnerV1 pid=2350) 'contents': [], (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'level': 'level0'}, (TaskRunnerV1 pid=2350) 'nsys': {'_target_': 'verl.utils.profiler.config.NsightToolConfig', (TaskRunnerV1 pid=2350) 'discrete': False}, (TaskRunnerV1 pid=2350) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2350) 'config_path': None, (TaskRunnerV1 pid=2350) 'stages': None, (TaskRunnerV1 pid=2350) 'steps': None, (TaskRunnerV1 pid=2350) 'strict': False}, (TaskRunnerV1 pid=2350) 'torch': {'_target_': 'verl.utils.profiler.config.TorchProfilerToolConfig', (TaskRunnerV1 pid=2350) 'contents': [], (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'schedule': {'_target_': 'verl.utils.profiler.config.TorchProfilerScheduleConfig', (TaskRunnerV1 pid=2350) 'active': 0, (TaskRunnerV1 pid=2350) 'repeat': 0, (TaskRunnerV1 pid=2350) 'skip_first': 0, (TaskRunnerV1 pid=2350) 'wait': 0, (TaskRunnerV1 pid=2350) 'warmup': 0}}, (TaskRunnerV1 pid=2350) 'torch_memory': {'_target_': 'verl.utils.profiler.config.TorchMemoryToolConfig', (TaskRunnerV1 pid=2350) 'stack_depth': 32, (TaskRunnerV1 pid=2350) 'trace_alloc_max_entries': 100000}}}, (TaskRunnerV1 pid=2350) 'rollout_n': 8, (TaskRunnerV1 pid=2350) 'shuffle': False, (TaskRunnerV1 pid=2350) 'strategy': 'fsdp', (TaskRunnerV1 pid=2350) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2350) 'use_dynamic_bsz': True}, (TaskRunnerV1 pid=2350) 'custom_reward_function': {'name': None, 'path': None}, (TaskRunnerV1 pid=2350) 'data': {'apply_chat_template_kwargs': {}, (TaskRunnerV1 pid=2350) 'audio_key': 'audios', (TaskRunnerV1 pid=2350) 'continuous_token': {'enable': False, 'model_family': 'auto'}, (TaskRunnerV1 pid=2350) 'custom_cls': {'name': None, 'path': None}, (TaskRunnerV1 pid=2350) 'dataloader_num_workers': 8, (TaskRunnerV1 pid=2350) 'filter_overlong_prompts': True, (TaskRunnerV1 pid=2350) 'filter_overlong_prompts_workers': 1, (TaskRunnerV1 pid=2350) 'function_tool_path': None, (TaskRunnerV1 pid=2350) 'gen_batch_size': None, (TaskRunnerV1 pid=2350) 'image_key': 'images', (TaskRunnerV1 pid=2350) 'image_patch_size': 14, (TaskRunnerV1 pid=2350) 'max_prompt_length': 128, (TaskRunnerV1 pid=2350) 'max_response_length': 128, (TaskRunnerV1 pid=2350) 'mm_processor_kwargs': {}, (TaskRunnerV1 pid=2350) 'prompt_key': 'prompt', (TaskRunnerV1 pid=2350) 'return_full_prompt': False, (TaskRunnerV1 pid=2350) 'return_multi_modal_inputs': True, (TaskRunnerV1 pid=2350) 'return_raw_chat': True, (TaskRunnerV1 pid=2350) 'return_raw_input_ids': False, (TaskRunnerV1 pid=2350) 'reward_fn_key': 'data_source', (TaskRunnerV1 pid=2350) 'seed': None, (TaskRunnerV1 pid=2350) 'shuffle': True, (TaskRunnerV1 pid=2350) 'tokenizer': None, (TaskRunnerV1 pid=2350) 'tool_config_path': None, (TaskRunnerV1 pid=2350) 'train_batch_size': 128, (TaskRunnerV1 pid=2350) 'train_files': '/data/char_count/rl/train.parquet', (TaskRunnerV1 pid=2350) 'train_max_samples': -1, (TaskRunnerV1 pid=2350) 'truncation': 'error', (TaskRunnerV1 pid=2350) 'trust_remote_code': False, (TaskRunnerV1 pid=2350) 'use_shm': False, (TaskRunnerV1 pid=2350) 'val_batch_size': None, (TaskRunnerV1 pid=2350) 'val_files': '/data/char_count/rl/test.parquet', (TaskRunnerV1 pid=2350) 'val_max_samples': -1, (TaskRunnerV1 pid=2350) 'validation_shuffle': False, (TaskRunnerV1 pid=2350) 'video_key': 'videos'}, (TaskRunnerV1 pid=2350) 'distillation': {'_target_': 'verl.workers.config.DistillationConfig', (TaskRunnerV1 pid=2350) 'distillation_loss': {'_target_': 'verl.workers.config.DistillationLossConfig', (TaskRunnerV1 pid=2350) 'clip_ratio': 0.2, (TaskRunnerV1 pid=2350) 'clip_ratio_high': 0.2, (TaskRunnerV1 pid=2350) 'clip_ratio_low': 0.2, (TaskRunnerV1 pid=2350) 'distillation_loss_coef': 1.0, (TaskRunnerV1 pid=2350) 'log_prob_min_clamp': None, (TaskRunnerV1 pid=2350) 'loss_max_clamp': None, (TaskRunnerV1 pid=2350) 'loss_mode': 'k3', (TaskRunnerV1 pid=2350) 'policy_loss_mode': 'vanilla', (TaskRunnerV1 pid=2350) 'topk': 32, (TaskRunnerV1 pid=2350) 'use_policy_gradient': False, (TaskRunnerV1 pid=2350) 'use_task_rewards': True}, (TaskRunnerV1 pid=2350) 'enabled': False, (TaskRunnerV1 pid=2350) 'n_gpus_per_node': 8, (TaskRunnerV1 pid=2350) 'nnodes': 0, (TaskRunnerV1 pid=2350) 'teacher_key': 'data_source', (TaskRunnerV1 pid=2350) 'teacher_models': {'teacher_model': {'_target_': 'verl.workers.config.DistillationTeacherModelConfig', (TaskRunnerV1 pid=2350) 'inference': {'_target_': 'verl.workers.config.RolloutConfig', (TaskRunnerV1 pid=2350) 'cudagraph_capture_sizes': None, (TaskRunnerV1 pid=2350) 'data_parallel_size': 1, (TaskRunnerV1 pid=2350) 'disable_log_stats': True, (TaskRunnerV1 pid=2350) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2350) 'enable_chunked_prefill': True, (TaskRunnerV1 pid=2350) 'enable_prefix_caching': True, (TaskRunnerV1 pid=2350) 'enforce_eager': True, (TaskRunnerV1 pid=2350) 'engine_kwargs': {}, (TaskRunnerV1 pid=2350) 'expert_parallel_size': 1, (TaskRunnerV1 pid=2350) 'free_cache_engine': True, (TaskRunnerV1 pid=2350) 'gpu_memory_utilization': 0.5, (TaskRunnerV1 pid=2350) 'limit_images': None, (TaskRunnerV1 pid=2350) 'load_format': 'auto', (TaskRunnerV1 pid=2350) 'max_model_len': None, (TaskRunnerV1 pid=2350) 'max_num_batched_tokens': 8192, (TaskRunnerV1 pid=2350) 'max_num_seqs': 1024, (TaskRunnerV1 pid=2350) 'name': 'vllm', (TaskRunnerV1 pid=2350) 'prompt_length': 128, (TaskRunnerV1 pid=2350) 'response_length': 128, (TaskRunnerV1 pid=2350) 'skip_tokenizer_init': True, (TaskRunnerV1 pid=2350) 'temperature': 1.0, (TaskRunnerV1 pid=2350) 'tensor_model_parallel_size': 2}, (TaskRunnerV1 pid=2350) 'key': None, (TaskRunnerV1 pid=2350) 'model_path': None, (TaskRunnerV1 pid=2350) 'num_replicas': 0}}}, (TaskRunnerV1 pid=2350) 'global_profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2350) 'global_tool_config': {'nsys': {'_target_': 'verl.utils.profiler.config.NsightToolConfig', (TaskRunnerV1 pid=2350) 'controller_nsight_options': {'cuda-graph-trace': 'graph', (TaskRunnerV1 pid=2350) 'cuda-memory-usage': 'true', (TaskRunnerV1 pid=2350) 'trace': 'cuda,nvtx,cublas,ucx'}, (TaskRunnerV1 pid=2350) 'discrete': False, (TaskRunnerV1 pid=2350) 'worker_nsight_options': {'capture-range': 'cudaProfilerApi', (TaskRunnerV1 pid=2350) 'capture-range-end': None, (TaskRunnerV1 pid=2350) 'cuda-graph-trace': 'graph', (TaskRunnerV1 pid=2350) 'cuda-memory-usage': 'true', (TaskRunnerV1 pid=2350) 'kill': 'none', (TaskRunnerV1 pid=2350) 'trace': 'cuda,nvtx,cublas,ucx'}}, (TaskRunnerV1 pid=2350) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2350) 'config_path': None, (TaskRunnerV1 pid=2350) 'stages': None, (TaskRunnerV1 pid=2350) 'steps': None, (TaskRunnerV1 pid=2350) 'strict': False}, (TaskRunnerV1 pid=2350) 'torch_memory': {'context': 'all', (TaskRunnerV1 pid=2350) 'kw_args': {}, (TaskRunnerV1 pid=2350) 'stack_depth': 32, (TaskRunnerV1 pid=2350) 'stacks': 'all', (TaskRunnerV1 pid=2350) 'trace_alloc_max_entries': 100000}}, (TaskRunnerV1 pid=2350) 'profile_continuous_steps': False, (TaskRunnerV1 pid=2350) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2350) 'steps': None, (TaskRunnerV1 pid=2350) 'tool': None}, (TaskRunnerV1 pid=2350) 'model_engine': 'dp', (TaskRunnerV1 pid=2350) 'ray_kwargs': {'ray_init': {'num_cpus': None, (TaskRunnerV1 pid=2350) 'runtime_env': {'py_executable': 'uv -v run ' (TaskRunnerV1 pid=2350) '--frozen ' (TaskRunnerV1 pid=2350) '--all-packages ' (TaskRunnerV1 pid=2350) '--extra vllm ' (TaskRunnerV1 pid=2350) '--extra fsdp'}}, (TaskRunnerV1 pid=2350) 'timeline_json_file': None}, (TaskRunnerV1 pid=2350) 'reward': {'_target_': 'verl.workers.config.RewardConfig', (TaskRunnerV1 pid=2350) 'custom_reward_function': {'name': 'compute_score', (TaskRunnerV1 pid=2350) 'path': '/run/reward_tolerant.py'}, (TaskRunnerV1 pid=2350) 'num_workers': 8, (TaskRunnerV1 pid=2350) 'reward_manager': {'_target_': 'verl.workers.config.reward.RewardManagerConfig', (TaskRunnerV1 pid=2350) 'module': {'_target_': 'verl.trainer.config.config.ModuleConfig', (TaskRunnerV1 pid=2350) 'name': 'custom_reward_manager', (TaskRunnerV1 pid=2350) 'path': None}, (TaskRunnerV1 pid=2350) 'name': 'naive', (TaskRunnerV1 pid=2350) 'source': 'register'}, (TaskRunnerV1 pid=2350) 'reward_model': {'_target_': 'verl.workers.config.RewardModelConfig', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'enable_resource_pool': False, (TaskRunnerV1 pid=2350) 'model_path': None, (TaskRunnerV1 pid=2350) 'n_gpus_per_node': 8, (TaskRunnerV1 pid=2350) 'nnodes': 0, (TaskRunnerV1 pid=2350) 'rollout': {'_target_': 'verl.workers.config.RolloutConfig', (TaskRunnerV1 pid=2350) 'cudagraph_capture_sizes': None, (TaskRunnerV1 pid=2350) 'data_parallel_size': 1, (TaskRunnerV1 pid=2350) 'disable_log_stats': True, (TaskRunnerV1 pid=2350) 'disaggregation': {'bootstrap_port': None, (TaskRunnerV1 pid=2350) 'decode_replicas': 1, (TaskRunnerV1 pid=2350) 'decode_tensor_model_parallel_size': None, (TaskRunnerV1 pid=2350) 'enabled': False, (TaskRunnerV1 pid=2350) 'ib_device': None, (TaskRunnerV1 pid=2350) 'prefill_replicas': 1, (TaskRunnerV1 pid=2350) 'transfer_backend': 'nixl'}, (TaskRunnerV1 pid=2350) 'do_sample': True, (TaskRunnerV1 pid=2350) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2350) 'enable_chunked_prefill': True, (TaskRunnerV1 pid=2350) 'enable_prefix_caching': True, (TaskRunnerV1 pid=2350) 'enforce_eager': True, (TaskRunnerV1 pid=2350) 'engine_kwargs': {}, (TaskRunnerV1 pid=2350) 'expert_parallel_size': 1, (TaskRunnerV1 pid=2350) 'free_cache_engine': True, (TaskRunnerV1 pid=2350) 'full_determinism': False, (TaskRunnerV1 pid=2350) 'gpu_memory_utilization': 0.5, (TaskRunnerV1 pid=2350) 'ignore_eos': False, (TaskRunnerV1 pid=2350) 'layered_summon': False, (TaskRunnerV1 pid=2350) 'limit_images': None, (TaskRunnerV1 pid=2350) 'load_format': 'auto', (TaskRunnerV1 pid=2350) 'max_model_len': None, (TaskRunnerV1 pid=2350) 'max_num_batched_tokens': 8192, (TaskRunnerV1 pid=2350) 'max_num_seqs': 1024, (TaskRunnerV1 pid=2350) 'mtp': None, (TaskRunnerV1 pid=2350) 'multi_stage_wake_up': False, (TaskRunnerV1 pid=2350) 'n': 1, (TaskRunnerV1 pid=2350) 'name': '???', (TaskRunnerV1 pid=2350) 'pipeline_model_parallel_size': 1, (TaskRunnerV1 pid=2350) 'prompt_length': 2048, (TaskRunnerV1 pid=2350) 'quantization': None, (TaskRunnerV1 pid=2350) 'quantization_config_file': None, (TaskRunnerV1 pid=2350) 'response_length': 2048, (TaskRunnerV1 pid=2350) 'scheduling_policy': 'fcfs', (TaskRunnerV1 pid=2350) 'seed': 42, (TaskRunnerV1 pid=2350) 'skip_tokenizer_init': False, (TaskRunnerV1 pid=2350) 'temperature': 1.0, (TaskRunnerV1 pid=2350) 'tensor_model_parallel_size': 2, (TaskRunnerV1 pid=2350) 'top_k': -1, (TaskRunnerV1 pid=2350) 'top_p': 1}}, (TaskRunnerV1 pid=2350) 'sandbox_fusion': {'_target_': 'verl.workers.config.SandboxFusionConfig', (TaskRunnerV1 pid=2350) 'max_concurrent': 64, (TaskRunnerV1 pid=2350) 'memory_limit_mb': 1024, (TaskRunnerV1 pid=2350) 'url': None}}, (TaskRunnerV1 pid=2350) 'reward_model': {'enable': None, (TaskRunnerV1 pid=2350) 'enable_resource_pool': None, (TaskRunnerV1 pid=2350) 'model': {'external_lib': None, (TaskRunnerV1 pid=2350) 'path': None, (TaskRunnerV1 pid=2350) 'trust_remote_code': None}, (TaskRunnerV1 pid=2350) 'n_gpus_per_node': None, (TaskRunnerV1 pid=2350) 'nnodes': None, (TaskRunnerV1 pid=2350) 'num_workers': None, (TaskRunnerV1 pid=2350) 'reward_loop_class_name': None, (TaskRunnerV1 pid=2350) 'reward_loop_module_path': None, (TaskRunnerV1 pid=2350) 'reward_loop_source': None, (TaskRunnerV1 pid=2350) 'reward_manager': None, (TaskRunnerV1 pid=2350) 'rollout': {'cudagraph_capture_sizes': None, (TaskRunnerV1 pid=2350) 'data_parallel_size': None, (TaskRunnerV1 pid=2350) 'disable_log_stats': None, (TaskRunnerV1 pid=2350) 'dtype': None, (TaskRunnerV1 pid=2350) 'enable_chunked_prefill': None, (TaskRunnerV1 pid=2350) 'enable_prefix_caching': None, (TaskRunnerV1 pid=2350) 'enforce_eager': None, (TaskRunnerV1 pid=2350) 'engine_kwargs': None, (TaskRunnerV1 pid=2350) 'expert_parallel_size': None, (TaskRunnerV1 pid=2350) 'free_cache_engine': None, (TaskRunnerV1 pid=2350) 'gpu_memory_utilization': None, (TaskRunnerV1 pid=2350) 'limit_images': None, (TaskRunnerV1 pid=2350) 'load_format': None, (TaskRunnerV1 pid=2350) 'max_model_len': None, (TaskRunnerV1 pid=2350) 'max_num_batched_tokens': None, (TaskRunnerV1 pid=2350) 'max_num_seqs': None, (TaskRunnerV1 pid=2350) 'name': None, (TaskRunnerV1 pid=2350) 'prompt_length': None, (TaskRunnerV1 pid=2350) 'response_length': None, (TaskRunnerV1 pid=2350) 'skip_tokenizer_init': None, (TaskRunnerV1 pid=2350) 'tensor_model_parallel_size': None}}, (TaskRunnerV1 pid=2350) 'sandbox_fusion': {'max_concurrent': None, (TaskRunnerV1 pid=2350) 'memory_limit_mb': None, (TaskRunnerV1 pid=2350) 'url': None},(raylet) DEBUG Found workspace root: `/workspace/verl` (raylet) DEBUG Adding root workspace member: `/workspace/verl` (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) (raylet) DEBUG Found project root: `/workspace/verl` (raylet) DEBUG No workspace root found, using project root (raylet) DEBUG Discovered project `verl` at: /workspace/verl (raylet) DEBUG No Python version file found in workspace: /workspace/verl (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata (raylet) DEBUG Checking for Python environment at: `.venv` (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` (raylet) DEBUG Skipping environment synchronization due to `--no-sync` (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=37761 --object-store-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:37513 --session-name=session_2026-11-01_16-48-32_811455_191 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=fa5e1a1d203a8a2c8e2dae1743d563c258a9a9fd2d3087d9f548d221 --worker-id=746e60dd56051376d0f0fc5c796a792e3f3fe290f17b38a0aa100a54 --worker-launch-time-ms=1793551740611 --node-id=e3087f4a64dbc01fd839260bc555975b183b534e612ab28707facf9a --runtime-env-hash=247405026` (raylet) DEBUG Spawned child 2570 in process group 1 (raylet) INFO Waiting to acquire exclusive lock for `/workspace/verl` at `/tmp/uv-50f1055beec85b7b.lock` (TaskRunnerV1 pid=2350) /workspace/verl/verl/trainer/ppo/v1/trainer_base.py:129: UserWarning: Disabled critic as algorithm.adv_estimator != gae. If it is not intended, please set critic.enable=True (TaskRunnerV1 pid=2350) self.use_critic = need_critic(self.config) (TaskRunnerV1 pid=2350) Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 9000 examples [00:00, 1435640.68 examples/s] (TaskRunnerV1 pid=2350) Setting TOKENIZERS_PARALLELISM=false for forked processes. (TaskRunnerV1 pid=2350) WARNING:2026-11-01 16:49:05,524:Setting TOKENIZERS_PARALLELISM=false for forked processes. (TaskRunnerV1 pid=2350) Filtering prompts longer than 128 tokens (num_proc=1): 0%| | 0/9000 [00:00=3.10, <3.13` from `requires-python` metadata [repeated 8x across cluster] (raylet) DEBUG Checking for Python environment at: `.venv` [repeated 8x across cluster] (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` [repeated 8x across cluster] (raylet) DEBUG Skipping environment synchronization due to `--no-sync` [repeated 8x across cluster] (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 [repeated 8x across cluster] (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=37761 --object-store-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:37513 --session-name=session_2026-11-01_16-48-32_811455_191 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=fa5e1a1d203a8a2c8e2dae1743d563c258a9a9fd2d3087d9f548d221 --worker-id=95c1985052e31e82e7a191e26b857866fbb5288d78f48fb34add87be --worker-launch-time-ms=1793551742150 --node-id=e3087f4a64dbc01fd839260bc555975b183b534e612ab28707facf9a --runtime-env-hash=247405026` [repeated 8x across cluster] (raylet) DEBUG Spawned child 2695 in process group 1 [repeated 8x across cluster] (TaskRunnerV1 pid=2350) Filtering prompts longer than 128 tokens (num_proc=1): 33%|███▎ | 3000/9000 [00:00<00:01, 5069.75 examples/s] (TaskRunnerV1 pid=2350) Filtering prompts longer than 128 tokens (num_proc=1): 56%|█████▌ | 5000/9000 [00:00<00:00, 7138.72 examples/s] (TaskRunnerV1 pid=2350) Filtering prompts longer than 128 tokens (num_proc=1): 78%|███████▊ | 7000/9000 [00:01<00:00, 8532.96 examples/s] (TaskRunnerV1 pid=2350) Filtering prompts longer than 128 tokens (num_proc=1): 100%|██████████| 9000/9000 [00:01<00:00, 9496.79 examples/s] (TaskRunnerV1 pid=2350) Filtering prompts longer than 128 tokens (num_proc=1): 100%|██████████| 9000/9000 [00:02<00:00, 3896.28 examples/s] (raylet) INFO Waiting to acquire exclusive lock for `/workspace/verl` at `/tmp/uv-50f1055beec85b7b.lock` [repeated 2x across cluster] (TaskRunnerV1 pid=2350) Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 1000 examples [00:00, 486860.59 examples/s] (TaskRunnerV1 pid=2350) Setting TOKENIZERS_PARALLELISM=false for forked processes. (TaskRunnerV1 pid=2350) WARNING:2026-11-01 16:49:08,169:Setting TOKENIZERS_PARALLELISM=false for forked processes. (TaskRunnerV1 pid=2350) Filtering prompts longer than 128 tokens (num_proc=1): 0%| | 0/1000 [00:00=3.10, <3.13` from `requires-python` metadata [repeated 2x across cluster] (raylet) DEBUG Checking for Python environment at: `.venv` [repeated 2x across cluster] (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` [repeated 2x across cluster] (raylet) DEBUG Skipping environment synchronization due to `--no-sync` [repeated 2x across cluster] (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 [repeated 2x across cluster] (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=37761 --object-store-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:37513 --session-name=session_2026-11-01_16-48-32_811455_191 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=fa5e1a1d203a8a2c8e2dae1743d563c258a9a9fd2d3087d9f548d221 --worker-id=bdf932271aaf83261ce7eacf7c2a8e3a52725d22566164d6ba556e69 --worker-launch-time-ms=1793551749308 --node-id=e3087f4a64dbc01fd839260bc555975b183b534e612ab28707facf9a --runtime-env-hash=922704060` [repeated 2x across cluster] (raylet) DEBUG Spawned child 3281 in process group 1 [repeated 2x across cluster] (WorkerDict pid=3281) Loading weights: 0%| | 0/272 [00:00=3.10, <3.13` from `requires-python` metadata (raylet) DEBUG Checking for Python environment at: `.venv` (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` (raylet) DEBUG Skipping environment synchronization due to `--no-sync` (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=37761 --object-store-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:37513 --session-name=session_2026-11-01_16-48-32_811455_191 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=fa5e1a1d203a8a2c8e2dae1743d563c258a9a9fd2d3087d9f548d221 --worker-id=d7d7b61ddd7869a27763d0b3335887d132719857fec97b9c32afd0e9 --worker-launch-time-ms=1793551760741 --node-id=e3087f4a64dbc01fd839260bc555975b183b534e612ab28707facf9a --runtime-env-hash=247405026` (raylet) DEBUG Spawned child 3445 in process group 1 (raylet) INFO Waiting to acquire exclusive lock for `/workspace/verl` at `/tmp/uv-50f1055beec85b7b.lock` (TaskRunnerV1 pid=2350) W1101 16:49:21.156000 2350 .venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py:140] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' (TaskRunnerV1 pid=2350) 'skip': {'_target_': 'verl.utils.skip.SkipManagerConfig', (TaskRunnerV1 pid=2350) 'async_rollout': {'action': 'cache', (TaskRunnerV1 pid=2350) 'dump_dir': '~/.verl/rollout_dump', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'steps': []}, (TaskRunnerV1 pid=2350) 'rollout': {'action': 'cache', (TaskRunnerV1 pid=2350) 'dump_dir': '~/.verl/rollout_dump', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'steps': []}, (TaskRunnerV1 pid=2350) 'rollout_tq': {'action': 'cache', (TaskRunnerV1 pid=2350) 'dump_dir': '~/.verl/rollout_dump', (TaskRunnerV1 pid=2350) 'enable': False, (TaskRunnerV1 pid=2350) 'steps': []}}, (TaskRunnerV1 pid=2350) 'trainer': {'balance_batch': True, (TaskRunnerV1 pid=2350) 'critic_warmup': 0, (TaskRunnerV1 pid=2350) 'default_hdfs_dir': None, (TaskRunnerV1 pid=2350) 'default_local_dir': '/checkpoints/char-count-smol135m', (TaskRunnerV1 pid=2350) 'del_local_ckpt_after_load': False, (TaskRunnerV1 pid=2350) 'device': 'cuda', (TaskRunnerV1 pid=2350) 'esi_redundant_time': 0, (TaskRunnerV1 pid=2350) 'experiment_name': 'char-count-smol135m-instruct-grpo-direct-1gpu', (TaskRunnerV1 pid=2350) 'log_val_generations': 0, (TaskRunnerV1 pid=2350) 'logger': ['console', 'mlflow'], (TaskRunnerV1 pid=2350) 'max_actor_ckpt_to_keep': None, (TaskRunnerV1 pid=2350) 'max_critic_ckpt_to_keep': None, (TaskRunnerV1 pid=2350) 'n_gpus_per_node': 1, (TaskRunnerV1 pid=2350) 'nnodes': 1, (TaskRunnerV1 pid=2350) 'project_name': 'verl_char_count', (TaskRunnerV1 pid=2350) 'ray_wait_register_center_timeout': 300, (TaskRunnerV1 pid=2350) 'resume_from_path': None, (TaskRunnerV1 pid=2350) 'resume_mode': 'auto', (TaskRunnerV1 pid=2350) 'rollout_data_dir': None, (TaskRunnerV1 pid=2350) 'save_freq': -1, (TaskRunnerV1 pid=2350) 'test_freq': 5, (TaskRunnerV1 pid=2350) 'total_epochs': 5, (TaskRunnerV1 pid=2350) 'total_training_steps': None, (TaskRunnerV1 pid=2350) 'use_v1': True, (TaskRunnerV1 pid=2350) 'v1': {'colocate_async': {'num_warmup_batches': 1}, (TaskRunnerV1 pid=2350) 'sampler': {'custom_sampler': {'name': None, 'path': None}, (TaskRunnerV1 pid=2350) 'max_off_policy_strategy': 'drop', (TaskRunnerV1 pid=2350) 'max_off_policy_threshold': 8, (TaskRunnerV1 pid=2350) 'sampler_kwargs': {}, (TaskRunnerV1 pid=2350) 'sync_refill_failed_groups': False}, (TaskRunnerV1 pid=2350) 'separate_async': {'num_warmup_batches': 1, (TaskRunnerV1 pid=2350) 'parameter_sync_step': 4}, (TaskRunnerV1 pid=2350) 'sync': {}, (TaskRunnerV1 pid=2350) 'trainer_mode': 'sync'}, (TaskRunnerV1 pid=2350) 'val_before_train': True, (TaskRunnerV1 pid=2350) 'val_only': False, (TaskRunnerV1 pid=2350) 'validation_data_dir': None}, (TaskRunnerV1 pid=2350) 'transfer_queue': {'backend': {'MooncakeStore': {'auto_init': False, (TaskRunnerV1 pid=2350) 'device_name': '', (TaskRunnerV1 pid=2350) 'global_segment_size': 4294967296, (TaskRunnerV1 pid=2350) 'local_buffer_size': 1073741824, (TaskRunnerV1 pid=2350) 'local_hostname': 'localhost', (TaskRunnerV1 pid=2350) 'master_server_address': 'localhost:50124', (TaskRunnerV1 pid=2350) 'metadata_server': 'localhost:50123', (TaskRunnerV1 pid=2350) 'protocol': 'tcp'}, (TaskRunnerV1 pid=2350) 'SimpleStorage': {'num_data_storage_units': 8, (TaskRunnerV1 pid=2350) 'total_storage_size': 100000}, (TaskRunnerV1 pid=2350) 'storage_backend': 'SimpleStorage'}, (TaskRunnerV1 pid=2350) 'enable': True, (TaskRunnerV1 pid=2350) 'metrics': {'enabled': False, 'port': 0}}} (TaskRunnerV1 pid=2350) Using dataset class: RLHFDataset (TaskRunnerV1 pid=2350) dataset len: 9000 (TaskRunnerV1 pid=2350) filter dataset len: 9000 (TaskRunnerV1 pid=2350) Using dataset class: RLHFDataset (TaskRunnerV1 pid=2350) dataset len: 1000 (TaskRunnerV1 pid=2350) filter dataset len: 1000 (TaskRunnerV1 pid=2350) colocated worker base class (WorkerDict pid=3281) Monkey patch state_dict in AutoModelForCausalLMWithValueHead. (WorkerDict pid=3281) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention (WorkerDict pid=3281) Skipping monkey patch for LlamaForCausalLM as use_fused_kernels is False or fused_kernels_backend is torch (WorkerDict pid=3281) LlamaForCausalLM contains 134.52M parameters (WorkerDict pid=3281) Before FSDP, memory allocated (GB): 0.00, memory reserved (GB): 0.00, device memory used/total (GB): 0.55/47.58 (WorkerDict pid=3281) After FSDP, memory allocated (GB): 0.25, memory reserved (GB): 0.34, device memory used/total (GB): 1.39/47.58 (WorkerDict pid=3281) Total steps: 350, num_warmup_steps: 0 (vLLMHttpServer pid=3893) ['serve', (vLLMHttpServer pid=3893) '/models/SmolLM2-135M-Instruct', (vLLMHttpServer pid=3893) '--dtype', (vLLMHttpServer pid=3893) 'bfloat16', (vLLMHttpServer pid=3893) '--load_format', (vLLMHttpServer pid=3893) 'dummy', (vLLMHttpServer pid=3893) '--distributed_executor_backend', (vLLMHttpServer pid=3893) 'mp', (vLLMHttpServer pid=3893) '--worker_extension_cls', (vLLMHttpServer pid=3893) 'verl.workers.rollout.vllm_rollout.utils.vLLMColocateWorkerExtension', (vLLMHttpServer pid=3893) '--max_model_len', (vLLMHttpServer pid=3893) '8192', (vLLMHttpServer pid=3893) '--max_num_seqs', (vLLMHttpServer pid=3893) '1024', (vLLMHttpServer pid=3893) '--max_num_batched_tokens', (vLLMHttpServer pid=3893) '8192', (vLLMHttpServer pid=3893) '--enable_prefix_caching', (vLLMHttpServer pid=3893) '--enable_sleep_mode', (vLLMHttpServer pid=3893) '--logprobs_mode', (vLLMHttpServer pid=3893) 'processed_logprobs', (vLLMHttpServer pid=3893) '--enforce_eager', (vLLMHttpServer pid=3893) '--gpu_memory_utilization', (vLLMHttpServer pid=3893) '0.7', (vLLMHttpServer pid=3893) '--disable_log_stats', (vLLMHttpServer pid=3893) '--tensor_parallel_size', (vLLMHttpServer pid=3893) '1', (vLLMHttpServer pid=3893) '--seed', (vLLMHttpServer pid=3893) '42', (vLLMHttpServer pid=3893) '--override_generation_config',(TaskRunnerV1 pid=2350) INFO:2026-11-01 16:49:52,656:checkpoint engine manager initialized (raylet) DEBUG Found workspace root: `/workspace/verl` [repeated 8x across cluster] (raylet) DEBUG Adding root workspace member: `/workspace/verl` [repeated 8x across cluster] (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` [repeated 8x across cluster] (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` [repeated 8x across cluster] (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) [repeated 8x across cluster] (raylet) DEBUG Found project root: `/workspace/verl` [repeated 8x across cluster] (raylet) DEBUG No workspace root found, using project root [repeated 8x across cluster] (raylet) DEBUG Discovered project `verl` at: /workspace/verl [repeated 8x across cluster] (raylet) DEBUG No Python version file found in workspace: /workspace/verl [repeated 8x across cluster] (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata [repeated 8x across cluster] (raylet) DEBUG Checking for Python environment at: `.venv` [repeated 8x across cluster] (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` [repeated 8x across cluster] (raylet) DEBUG Skipping environment synchronization due to `--no-sync` [repeated 8x across cluster] (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 [repeated 8x across cluster] (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=37761 --object-store-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:37513 --session-name=session_2026-11-01_16-48-32_811455_191 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=fa5e1a1d203a8a2c8e2dae1743d563c258a9a9fd2d3087d9f548d221 --worker-id=757fd0d971f80a7adcda124238c53452d18bd770d04c1e192fd1cd65 --worker-launch-time-ms=1793551762222 --node-id=e3087f4a64dbc01fd839260bc555975b183b534e612ab28707facf9a --runtime-env-hash=-1119239678` [repeated 8x across cluster] (raylet) DEBUG Spawned child 3893 in process group 1 [repeated 8x across cluster] (TaskRunnerV1 pid=2350) INFO:2026-11-01 16:49:52,760:Training from scratch (TaskRunnerV1 pid=2350) INFO:2026-11-01 16:49:52,760:all initialize finished, ready to fit (WorkerDict pid=3281) /workspace/verl/verl/plugin/platform/platform_cuda.py:99: FutureWarning: torch.cuda._set_allocator_settings is deprecated. Use torch._C._accelerator_setAllocatorSettings instead. (WorkerDict pid=3281) torch.cuda.memory._set_allocator_settings(settings) (TaskRunnerV1 pid=2350) INFO:2026-11-01 16:49:54,461:Using MLFlow tracking URI: http://:5000/mlflow (TaskRunnerV1 pid=2350) 2026/11/01 16:49:54 INFO mlflow.tracking.fluent: Experiment with name 'verl_char_count' does not exist. Creating a new experiment. (GlobalRequestLoadBalancer pid=4391) W1101 16:49:56.984000 4391 .venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py:140] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' (pid=4463) WARNING:2026-11-01 16:49:58,634:Platform 'nvidia' (PlatformCUDA) is registered but not available. This may be due to this ray actor being a CPU-only actor. (pid=4463) /workspace/verl/verl/workers/engine/mindspeed/transformer_impl.py:24: UserWarning: NPU not support router replay for now. (pid=4463) from verl.utils.megatron.router_replay_patch import RouterReplay (raylet) DEBUG Found workspace root: `/workspace/verl` [repeated 9x across cluster] (raylet) DEBUG Adding root workspace member: `/workspace/verl` [repeated 9x across cluster] (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` [repeated 9x across cluster] (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` [repeated 9x across cluster] (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) [repeated 9x across cluster] (raylet) DEBUG Found project root: `/workspace/verl` [repeated 9x across cluster] (raylet) DEBUG No workspace root found, using project root [repeated 9x across cluster] (raylet) DEBUG Discovered project `verl` at: /workspace/verl [repeated 9x across cluster] (raylet) DEBUG No Python version file found in workspace: /workspace/verl [repeated 9x across cluster] (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata [repeated 9x across cluster] (raylet) DEBUG Checking for Python environment at: `.venv` [repeated 9x across cluster] (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` [repeated 9x across cluster] (raylet) DEBUG Skipping environment synchronization due to `--no-sync` [repeated 9x across cluster] (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 [repeated 9x across cluster] (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=37761 --object-store-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_16-48-32_811455_191/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:37513 --session-name=session_2026-11-01_16-48-32_811455_191 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=fa5e1a1d203a8a2c8e2dae1743d563c258a9a9fd2d3087d9f548d221 --worker-id=229d8b71b64b3e5d11ae4d7a771c0637167c1899423b35f2757777d0 --worker-launch-time-ms=1793551794221 --node-id=e3087f4a64dbc01fd839260bc555975b183b534e612ab28707facf9a --runtime-env-hash=247405026` [repeated 9x across cluster] (raylet) DEBUG Spawned child 4505 in process group 1 [repeated 9x across cluster] (AgentLoopWorkerTQ pid=4493) Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead. (AgentLoopWorkerTQ pid=4505) W1101 16:50:00.189000 4505 .venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py:140] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' [repeated 8x across cluster] (TaskRunnerV1 pid=2350) WARNING:2026-11-01 16:50:06,079:[MLflow] Metric key 'val-aux/char_count/reward/mean@1' sanitized to 'val-aux/char_count/reward/mean_at_1' due to invalid characters. (TaskRunnerV1 pid=2350) WARNING:2026-11-01 16:50:06,079:[MLflow] Metric key 'val-core/char_count/acc/mean@1' sanitized to 'val-core/char_count/acc/mean_at_1' due to invalid characters. (RewardLoopWorker pid=3464) WARNING:2026-11-01 16:50:02,169:Platform 'nvidia' (PlatformCUDA) is registered but not available. This may be due to this ray actor being a CPU-only actor. [repeated 15x across cluster] (pid=4505) /workspace/verl/verl/workers/engine/mindspeed/transformer_impl.py:24: UserWarning: NPU not support router replay for now. [repeated 7x across cluster] (pid=4505) from verl.utils.megatron.router_replay_patch import RouterReplay [repeated 7x across cluster] (AgentLoopWorkerTQ pid=4505) Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead. [repeated 7x across cluster] (TaskRunnerV1 pid=2350) Training Progress: 0%| | 0/350 [00:00:35339'] (TaskRunnerV1 pid=2350) Checkpoint tracker file does not exist: /checkpoints/char-count-smol135m/latest_checkpointed_iteration.txt (AgentLoopWorkerTQ pid=4493) Using dataset class: RLHFDataset (vLLMHttpServer pid=3893) WARNING 11-01 16:50:01 [input_processor.py:282] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat(). (vLLMHttpServer pid=3893) (Worker pid=4081) WARNING 11-01 16:50:01 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _topk_log_softmax_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. (TaskRunnerV1 pid=2350) ("Initial validation metrics: {'val-aux/char_count/reward/mean@1': " (TaskRunnerV1 pid=2350) "np.float64(0.026), 'val-core/char_count/acc/mean@1': np.float64(0.026), " (TaskRunnerV1 pid=2350) "'val-aux/num_turns/min': np.int64(2), 'val-aux/num_turns/max': np.int64(2), " (TaskRunnerV1 pid=2350) "'val-aux/num_turns/mean': np.float64(2.0)}") (AgentLoopWorkerTQ pid=4505) Using dataset class: RLHFDataset [repeated 7x across cluster] (TaskRunnerV1 pid=2350) step:0 - val-aux/char_count/reward/mean@1:np.float64(0.026) - val-core/char_count/acc/mean@1:np.float64(0.026) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) (TaskRunnerV1 pid=2350) step:1 - global_seqlen/min:160552.0 - global_seqlen/max:160552.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160552.0 - global_seqlen/balanced_max:160552.0 - global_seqlen/mean:160552.0 - actor/entropy:1.6828556060791016 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2509966790676117 - training/rollout_probs_diff_mean:0.006744848098605871 - training/rollout_probs_diff_std:0.011070027947425842 - training/rollout_actor_probs_pearson_corr:0.9994683265686035 - rollout_corr/training_ppl:10.251370429992676 - rollout_corr/training_log_ppl:1.73691725730896 - rollout_corr/kl:0.0018430262571200728 - rollout_corr/k3_kl:0.0016683199210092425 - rollout_corr/rollout_ppl:10.21653938293457 - rollout_corr/rollout_log_ppl:1.7346125841140747 - rollout_corr/log_ppl_diff:0.0023046073038131 - rollout_corr/log_ppl_abs_diff:0.00609780615195632 - rollout_corr/log_ppl_diff_max:0.11261844635009766 - rollout_corr/log_ppl_diff_min:-0.053250789642333984 - rollout_corr/ppl_ratio:1.0023517608642578 - rollout_corr/chi2_token:0.0030112266540527344 - rollout_corr/chi2_seq:0.31982481479644775 - actor/pg_clipfrac:0.000869146342665772 - actor/ppo_kl:-5.212990136745552e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.007752778434223728 - actor/loss:0.007752778707072139 - actor/grad_norm:0.2086181640625 - actor/perf/max_memory_allocated_gb:4.053352355957031 - actor/perf/max_memory_reserved_gb:4.0546875 - actor/perf/cpu_memory_used_gb:27.6493501663208 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:1 - training/epoch:0 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.001810820889659226 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.001810820889659226 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.2890625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5048828125 - response_length_non_aborted/mean:94.2890625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5048828125 - response/aborted_ratio:0.0 - prompt_length/mean:62.5 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.112515495995467 - timing_s/old_log_prob:3.820389114000136 - timing_s/adv:0.18248334399686428 - timing_s/update_actor:6.014792105001106 - timing_s/update_weights:0.9912674360020901 - timing_s/step:15.421263799005828 - timing_per_token_ms/adv:0.0011365996312525802 - timing_per_token_ms/update_actor:0.03746320260726186 - timing_per_token_ms/gen:0.042593788797699345 - perf/total_num_tokens:160552 - perf/time_per_step:15.421263799005828 - perf/throughput:10411.07927940059 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:2 - global_seqlen/min:161087.0 - global_seqlen/max:161087.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161087.0 - global_seqlen/balanced_max:161087.0 - global_seqlen/mean:161087.0 - actor/entropy:1.6515648365020752 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2924582064151764 - training/rollout_probs_diff_mean:0.006933975499123335 - training/rollout_probs_diff_std:0.011644684709608555 - training/rollout_actor_probs_pearson_corr:0.9994187951087952 - rollout_corr/training_ppl:10.348248481750488 - rollout_corr/training_log_ppl:1.7195683717727661 - rollout_corr/kl:0.0017309278482571244 - rollout_corr/k3_kl:0.001717328792437911 - rollout_corr/rollout_ppl:10.291711807250977 - rollout_corr/rollout_log_ppl:1.7175066471099854 - rollout_corr/log_ppl_diff:0.002061667153611779 - rollout_corr/log_ppl_abs_diff:0.006612442433834076 - rollout_corr/log_ppl_diff_max:0.1707620620727539 - rollout_corr/log_ppl_diff_min:-0.09662151336669922 - rollout_corr/ppl_ratio:1.0021299123764038 - rollout_corr/chi2_token:0.0034155845642089844 - rollout_corr/chi2_seq:0.4543750286102295 - actor/pg_clipfrac:0.0007280550653376849 - actor/ppo_kl:0.00026701112210503197 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0017913480114657432 - actor/loss:0.001791347865946591 - actor/grad_norm:0.20269775390625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:28.570865154266357 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:2 - training/epoch:0 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.005519572179764509 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.005519572179764509 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.2958984375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4921875 - response_length_non_aborted/mean:94.2958984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4921875 - response/aborted_ratio:0.0 - prompt_length/mean:63.015625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.151631993998308 - timing_s/old_log_prob:2.3890164330005064 - timing_s/adv:0.18510605199844576 - timing_s/update_actor:4.835607852997782 - timing_s/update_weights:0.9890488200035179 - timing_s/step:12.733017739003117 - timing_per_token_ms/adv:0.0011491060855217724 - timing_per_token_ms/update_actor:0.030018610148539498 - timing_per_token_ms/gen:0.04299580561105964 - perf/total_num_tokens:161087 - perf/time_per_step:12.733017739003117 - perf/throughput:12651.125075131773 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 1%| | 2/350 [00:28<1:22:23, 14.21s/it] (TaskRunnerV1 pid=2350) Training Progress: 1%| | 3/350 [00:41<1:18:10, 13.52s/it] (TaskRunnerV1 pid=2350) Training Progress: 1%| | 4/350 [00:54<1:17:11, 13.39s/it] (TaskRunnerV1 pid=2350) step:3 - global_seqlen/min:155111.0 - global_seqlen/max:155111.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155111.0 - global_seqlen/balanced_max:155111.0 - global_seqlen/mean:155111.0 - actor/entropy:1.685996651649475 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2282060980796814 - training/rollout_probs_diff_mean:0.006768322084099054 - training/rollout_probs_diff_std:0.011141380295157433 - training/rollout_actor_probs_pearson_corr:0.9994645118713379 - rollout_corr/training_ppl:10.428398132324219 - rollout_corr/training_log_ppl:1.7715911865234375 - rollout_corr/kl:0.0015527765499427915 - rollout_corr/k3_kl:0.0016450014663860202 - rollout_corr/rollout_ppl:10.419332504272461 - rollout_corr/rollout_log_ppl:1.770029902458191 - rollout_corr/log_ppl_diff:0.0015610640402883291 - rollout_corr/log_ppl_abs_diff:0.006322104949504137 - rollout_corr/log_ppl_diff_max:0.06808757781982422 - rollout_corr/log_ppl_diff_min:-0.08663702011108398 - rollout_corr/ppl_ratio:1.0016067028045654 - rollout_corr/chi2_token:0.0035147666931152344 - rollout_corr/chi2_seq:0.37658190727233887 - actor/pg_clipfrac:0.0008774464283002695 - actor/ppo_kl:-0.0001173806712510365 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0035422710061538965 - actor/loss:-0.0035422707442194223 - actor/grad_norm:0.267333984375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:28.6194167137146 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:3 - training/epoch:0 - critic/score/mean:0.037109375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.037109375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.008877300657331944 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.008877300657331944 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:90.3349609375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4765625 - response_length_non_aborted/mean:90.3349609375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4765625 - response/aborted_ratio:0.0 - prompt_length/mean:61.140625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.128054805005377 - timing_s/old_log_prob:2.315912345999095 - timing_s/adv:0.19096302300022217 - timing_s/update_actor:4.77015630799724 - timing_s/update_weights:0.9744760100002168 - timing_s/step:12.561714447001577 - timing_per_token_ms/adv:0.001231137849670379 - timing_per_token_ms/update_actor:0.030753178742946922 - timing_per_token_ms/gen:0.04462617217825775 - perf/total_num_tokens:155111 - perf/time_per_step:12.561714447001577 - perf/throughput:12347.91641335425 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:4 - global_seqlen/min:156068.0 - global_seqlen/max:156068.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156068.0 - global_seqlen/balanced_max:156068.0 - global_seqlen/mean:156068.0 - actor/entropy:1.6760444641113281 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2397795021533966 - training/rollout_probs_diff_mean:0.0067677004262804985 - training/rollout_probs_diff_std:0.011178458109498024 - training/rollout_actor_probs_pearson_corr:0.9994618892669678 - rollout_corr/training_ppl:11.07052993774414 - rollout_corr/training_log_ppl:1.7601643800735474 - rollout_corr/kl:0.0016295098466798663 - rollout_corr/k3_kl:0.0016609587473794818 - rollout_corr/rollout_ppl:10.977937698364258 - rollout_corr/rollout_log_ppl:1.7583324909210205 - rollout_corr/log_ppl_diff:0.001831766334362328 - rollout_corr/log_ppl_abs_diff:0.007064449600875378 - rollout_corr/log_ppl_diff_max:0.2331991195678711 - rollout_corr/log_ppl_diff_min:-0.051862239837646484 - rollout_corr/ppl_ratio:1.001914381980896 - rollout_corr/chi2_token:0.0033909082412719727 - rollout_corr/chi2_seq:0.6429605484008789 - actor/pg_clipfrac:0.0007871808453880174 - actor/ppo_kl:-0.00022636402369850356 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.004695781419286504 - actor/loss:-0.004695781390182674 - actor/grad_norm:0.2689208984375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:28.695640087127686 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:4 - training/epoch:0 - critic/score/mean:0.0322265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0322265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00094510184135288 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.00094510184135288 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.58203125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.494140625 - response_length_non_aborted/mean:91.58203125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.494140625 - response/aborted_ratio:0.0 - prompt_length/mean:60.828125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.130787934998807 - timing_s/old_log_prob:2.529512747001718 - timing_s/adv:0.25470043999666814 - timing_s/update_actor:4.819335533000412 - timing_s/update_weights:0.9772315520021948 - timing_s/step:13.058900675001496 - timing_per_token_ms/adv:0.001631983750651435 - timing_per_token_ms/update_actor:0.030879716104521185 - timing_per_token_ms/gen:0.04404764272764776 - perf/total_num_tokens:156068 - perf/time_per_step:13.058900675001496 - perf/throughput:11951.082551593274 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:5 - global_seqlen/min:159360.0 - global_seqlen/max:159360.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159360.0 - global_seqlen/balanced_max:159360.0 - global_seqlen/mean:159360.0 - actor/entropy:1.7061960697174072 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.1825663149356842 - training/rollout_probs_diff_mean:0.006775696761906147 - training/rollout_probs_diff_std:0.011050350964069366 - training/rollout_actor_probs_pearson_corr:0.9994674324989319 - rollout_corr/training_ppl:9.128713607788086 - rollout_corr/training_log_ppl:1.7580546140670776 - rollout_corr/kl:0.0015479361172765493 - rollout_corr/k3_kl:0.001646767952479422 - rollout_corr/rollout_ppl:9.101883888244629 - rollout_corr/rollout_log_ppl:1.7562196254730225 - rollout_corr/log_ppl_diff:0.0018347785808146 - rollout_corr/log_ppl_abs_diff:0.0058808051981031895 - rollout_corr/log_ppl_diff_max:0.06773018836975098 - rollout_corr/log_ppl_diff_min:-0.05665254592895508 - rollout_corr/ppl_ratio:1.0018739700317383 - rollout_corr/chi2_token:0.003496408462524414 - rollout_corr/chi2_seq:0.41507935523986816 - actor/pg_clipfrac:0.0008835983856746072 - actor/ppo_kl:0.00010325996208848662 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.00023364537628367543 - actor/loss:0.00023364555090665817 - actor/grad_norm:0.2415771484375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:28.725246906280518 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.028) - val-core/char_count/acc/mean@1:np.float64(0.028) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:5 - training/epoch:0 - critic/score/mean:0.0341796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0341796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0006093317060731351 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0006093317060731351 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:95.046875 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.509765625 - response_length_non_aborted/mean:95.046875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.509765625 - response/aborted_ratio:0.0 - prompt_length/mean:60.578125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.126539172997582 - timing_s/old_log_prob:2.3963849270003266 - timing_s/adv:0.3864115530013805 - timing_s/update_actor:4.841955140997015 - timing_s/update_weights:0.9929970980010694 - timing_s/step:12.923090588999912 - timing_s/testing:4.590122923997114 - timing_per_token_ms/adv:0.0024247712914243257 - timing_per_token_ms/update_actor:0.030383754649830667 - timing_per_token_ms/gen:0.04239827360058341 - perf/total_num_tokens:159360 - perf/time_per_step:12.923090588999912 - perf/throughput:12331.415531176935 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 1%|▏ | 5/350 [01:12<1:25:47, 14.92s/it] (TaskRunnerV1 pid=2350) Training Progress: 2%|▏ | 6/350 [01:25<1:21:56, 14.29s/it] (TaskRunnerV1 pid=2350) Training Progress: 2%|▏ | 7/350 [01:38<1:19:19, 13.88s/it] (TaskRunnerV1 pid=2350) step:6 - global_seqlen/min:160168.0 - global_seqlen/max:160168.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160168.0 - global_seqlen/balanced_max:160168.0 - global_seqlen/mean:160168.0 - actor/entropy:1.6418648958206177 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3484281301498413 - training/rollout_probs_diff_mean:0.006864531431347132 - training/rollout_probs_diff_std:0.011280273087322712 - training/rollout_actor_probs_pearson_corr:0.9994475245475769 - rollout_corr/training_ppl:9.038430213928223 - rollout_corr/training_log_ppl:1.6962339878082275 - rollout_corr/kl:0.0016664995346218348 - rollout_corr/k3_kl:0.0016275342786684632 - rollout_corr/rollout_ppl:8.938982963562012 - rollout_corr/rollout_log_ppl:1.6941509246826172 - rollout_corr/log_ppl_diff:0.0020828393753618 - rollout_corr/log_ppl_abs_diff:0.005798060446977615 - rollout_corr/log_ppl_diff_max:0.09838342666625977 - rollout_corr/log_ppl_diff_min:-0.03480076789855957 - rollout_corr/ppl_ratio:1.0021222829818726 - rollout_corr/chi2_token:0.003186345100402832 - rollout_corr/chi2_seq:0.37070727348327637 - actor/pg_clipfrac:0.000795106723671779 - actor/ppo_kl:4.137259429626283e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005759028885222506 - actor/loss:-0.00575902865966782 - actor/grad_norm:0.216827392578125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:28.850162029266357 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:6 - training/epoch:0 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0017406801925972104 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0017406801925972104 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:95.8671875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.537109375 - response_length_non_aborted/mean:95.8671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.537109375 - response/aborted_ratio:0.0 - prompt_length/mean:60.546875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.340247118001571 - timing_s/old_log_prob:2.3586757669982035 - timing_s/adv:0.2388667940031155 - timing_s/update_actor:4.8479677270006505 - timing_s/update_weights:0.9849514310044469 - timing_s/step:12.950122098001884 - timing_per_token_ms/adv:0.001491351543398903 - timing_per_token_ms/update_actor:0.03026801687603423 - timing_per_token_ms/gen:0.04421244313830954 - perf/total_num_tokens:160168 - perf/time_per_step:12.950122098001884 - perf/throughput:12368.068716874324 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:7 - global_seqlen/min:159176.0 - global_seqlen/max:159176.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159176.0 - global_seqlen/balanced_max:159176.0 - global_seqlen/mean:159176.0 - actor/entropy:1.6403225660324097 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.22938865423202515 - training/rollout_probs_diff_mean:0.006711718626320362 - training/rollout_probs_diff_std:0.010987079702317715 - training/rollout_actor_probs_pearson_corr:0.9994749426841736 - rollout_corr/training_ppl:9.609396934509277 - rollout_corr/training_log_ppl:1.721153974533081 - rollout_corr/kl:0.0014749739784747362 - rollout_corr/k3_kl:0.0016187988221645355 - rollout_corr/rollout_ppl:9.551936149597168 - rollout_corr/rollout_log_ppl:1.7193115949630737 - rollout_corr/log_ppl_diff:0.0018425446469336748 - rollout_corr/log_ppl_abs_diff:0.006112054456025362 - rollout_corr/log_ppl_diff_max:0.0703115463256836 - rollout_corr/log_ppl_diff_min:-0.06024575233459473 - rollout_corr/ppl_ratio:1.0018874406814575 - rollout_corr/chi2_token:0.003548145294189453 - rollout_corr/chi2_seq:0.4555387496948242 - actor/pg_clipfrac:0.0008157745754336551 - actor/ppo_kl:-8.124066084747031e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0010158869336009957 - actor/loss:-0.0010158870718441904 - actor/grad_norm:0.24853515625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:28.85326623916626 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:7 - training/epoch:0 - critic/score/mean:0.02734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.02734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0014901539543643594 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0014901539543643594 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.7578125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4853515625 - response_length_non_aborted/mean:93.7578125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4853515625 - response/aborted_ratio:0.0 - prompt_length/mean:61.6875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.144345038999745 - timing_s/old_log_prob:2.5616290899997693 - timing_s/adv:0.18715742300264537 - timing_s/update_actor:4.829483203000564 - timing_s/update_weights:0.979650834997301 - timing_s/step:12.877361443999689 - timing_per_token_ms/adv:0.0011757892081887054 - timing_per_token_ms/update_actor:0.03034052371589036 - timing_per_token_ms/gen:0.043166663600947265 - perf/total_num_tokens:159176 - perf/time_per_step:12.877361443999689 - perf/throughput:12360.91731153274 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:8 - global_seqlen/min:158957.0 - global_seqlen/max:158957.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158957.0 - global_seqlen/balanced_max:158957.0 - global_seqlen/mean:158957.0 - actor/entropy:1.6565059423446655 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.29775017499923706 - training/rollout_probs_diff_mean:0.006818692199885845 - training/rollout_probs_diff_std:0.011181221343576908 - training/rollout_actor_probs_pearson_corr:0.9994586706161499 - rollout_corr/training_ppl:9.148656845092773 - rollout_corr/training_log_ppl:1.7025874853134155 - rollout_corr/kl:0.001598803442902863 - rollout_corr/k3_kl:0.0016479118494316936 - rollout_corr/rollout_ppl:9.141473770141602 - rollout_corr/rollout_log_ppl:1.7011001110076904 - rollout_corr/log_ppl_diff:0.0014874812914058566 - rollout_corr/log_ppl_abs_diff:0.0057286047376692295 - rollout_corr/log_ppl_diff_max:0.07625579833984375 - rollout_corr/log_ppl_diff_min:-0.03841996192932129 - rollout_corr/ppl_ratio:1.0015232563018799 - rollout_corr/chi2_token:0.00342714786529541 - rollout_corr/chi2_seq:0.3396620750427246 - actor/pg_clipfrac:0.0005802587911603041 - actor/ppo_kl:3.038034973693962e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.004669647038099356 - actor/loss:-0.004669647663831711 - actor/grad_norm:0.16943359375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:28.8967866897583 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:8 - training/epoch:0 - critic/score/mean:0.0205078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0205078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.007374657783657312 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.935412585735321 - critic/returns/mean:0.007374657783657312 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.935412585735321 - response_length/mean:93.2470703125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4931640625 - response_length_non_aborted/mean:93.2470703125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4931640625 - response/aborted_ratio:0.0 - prompt_length/mean:61.984375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.148211929001263 - timing_s/old_log_prob:2.5543867019950994 - timing_s/adv:0.18873318000260042 - timing_s/update_actor:4.829685082004289 - timing_s/update_weights:0.9886337169955368 - timing_s/step:12.889336965999973 - timing_per_token_ms/adv:0.001187322231814896 - timing_per_token_ms/update_actor:0.030383594821267946 - timing_per_token_ms/gen:0.04344359772740497 - perf/total_num_tokens:158957 - perf/time_per_step:12.889336965999973 - perf/throughput:12332.441957201007 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 2%|▏ | 8/350 [01:51<1:17:32, 13.60s/it] (TaskRunnerV1 pid=2350) Training Progress: 3%|▎ | 9/350 [02:04<1:16:18, 13.43s/it] (TaskRunnerV1 pid=2350) Training Progress: 3%|▎ | 10/350 [02:22<1:23:19, 14.71s/it] (TaskRunnerV1 pid=2350) step:9 - global_seqlen/min:159359.0 - global_seqlen/max:159359.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159359.0 - global_seqlen/balanced_max:159359.0 - global_seqlen/mean:159359.0 - actor/entropy:1.6631362438201904 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.39990562200546265 - training/rollout_probs_diff_mean:0.006743613630533218 - training/rollout_probs_diff_std:0.011316061019897461 - training/rollout_actor_probs_pearson_corr:0.9994515776634216 - rollout_corr/training_ppl:9.862996101379395 - rollout_corr/training_log_ppl:1.7363625764846802 - rollout_corr/kl:0.0018000341951847076 - rollout_corr/k3_kl:0.0016295224195346236 - rollout_corr/rollout_ppl:9.86054515838623 - rollout_corr/rollout_log_ppl:1.7351179122924805 - rollout_corr/log_ppl_diff:0.0012448129709810019 - rollout_corr/log_ppl_abs_diff:0.00637327553704381 - rollout_corr/log_ppl_diff_max:0.04526412487030029 - rollout_corr/log_ppl_diff_min:-0.18230366706848145 - rollout_corr/ppl_ratio:1.0013129711151123 - rollout_corr/chi2_token:0.0029556751251220703 - rollout_corr/chi2_seq:0.3043174743652344 - actor/pg_clipfrac:0.0007494958093351063 - actor/ppo_kl:-0.00031098610559846895 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.004295121445466066 - actor/loss:-0.004295121121685952 - actor/grad_norm:0.21673583984375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:28.986693382263184 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:9 - training/epoch:0 - critic/score/mean:0.03125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.03125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.006024689879268408 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.006024689879268408 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.5458984375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.517578125 - response_length_non_aborted/mean:93.5458984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.517578125 - response/aborted_ratio:0.0 - prompt_length/mean:62.078125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.135663132998161 - timing_s/old_log_prob:2.358476877998328 - timing_s/adv:0.3858166479985812 - timing_s/update_actor:4.877395550000074 - timing_s/update_weights:0.9796140819962602 - timing_s/step:12.923306515003787 - timing_per_token_ms/adv:0.002421053395155474 - timing_per_token_ms/update_actor:0.030606338832447955 - timing_per_token_ms/gen:0.043173817300144704 - perf/total_num_tokens:159359 - perf/time_per_step:12.923306515003787 - perf/throughput:12331.132115065624 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:10 - global_seqlen/min:161126.0 - global_seqlen/max:161126.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161126.0 - global_seqlen/balanced_max:161126.0 - global_seqlen/mean:161126.0 - actor/entropy:1.7392832040786743 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.24386581778526306 - training/rollout_probs_diff_mean:0.006708321627229452 - training/rollout_probs_diff_std:0.011068370193243027 - training/rollout_actor_probs_pearson_corr:0.999473512172699 - rollout_corr/training_ppl:11.010246276855469 - rollout_corr/training_log_ppl:1.7998080253601074 - rollout_corr/kl:0.0017282706685364246 - rollout_corr/k3_kl:0.0016798273427411914 - rollout_corr/rollout_ppl:10.998577117919922 - rollout_corr/rollout_log_ppl:1.7978023290634155 - rollout_corr/log_ppl_diff:0.0020057084038853645 - rollout_corr/log_ppl_abs_diff:0.006130394991487265 - rollout_corr/log_ppl_diff_max:0.05690765380859375 - rollout_corr/log_ppl_diff_min:-0.06028735637664795 - rollout_corr/ppl_ratio:1.002048134803772 - rollout_corr/chi2_token:0.003266453742980957 - rollout_corr/chi2_seq:0.4142637252807617 - actor/pg_clipfrac:0.0006570514524355531 - actor/ppo_kl:0.00015369303758442734 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005573194000135118 - actor/loss:-0.00557319400832057 - actor/grad_norm:0.25335693359375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.016075611114502 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.027) - val-core/char_count/acc/mean@1:np.float64(0.027) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:10 - training/epoch:0 - critic/score/mean:0.029296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.029296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.004865369759500027 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.004865369759500027 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.583984375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4990234375 - response_length_non_aborted/mean:94.583984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4990234375 - response/aborted_ratio:0.0 - prompt_length/mean:62.765625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.147770301999117 - timing_s/old_log_prob:2.373550787997374 - timing_s/adv:0.38437205299851485 - timing_s/update_actor:4.764639096996689 - timing_s/update_weights:0.9847125949963811 - timing_s/step:12.842049081002187 - timing_s/testing:4.589477786998032 - timing_per_token_ms/adv:0.002385537113802334 - timing_per_token_ms/update_actor:0.029570889223320188 - timing_per_token_ms/gen:0.04282497678979822 - perf/total_num_tokens:161126 - perf/time_per_step:12.842049081002187 - perf/throughput:12546.751611342215 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:11 - global_seqlen/min:155537.0 - global_seqlen/max:155537.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155537.0 - global_seqlen/balanced_max:155537.0 - global_seqlen/mean:155537.0 - actor/entropy:1.7265692949295044 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.24335914850234985 - training/rollout_probs_diff_mean:0.006765462923794985 - training/rollout_probs_diff_std:0.01123965997248888 - training/rollout_actor_probs_pearson_corr:0.9994576573371887 - rollout_corr/training_ppl:11.248533248901367 - rollout_corr/training_log_ppl:1.7991385459899902 - rollout_corr/kl:0.0018212407594546676 - rollout_corr/k3_kl:0.0017009833827614784 - rollout_corr/rollout_ppl:11.229270935058594 - rollout_corr/rollout_log_ppl:1.7968767881393433 - rollout_corr/log_ppl_diff:0.002261569956317544 - rollout_corr/log_ppl_abs_diff:0.006813503336161375 - rollout_corr/log_ppl_diff_max:0.08765840530395508 - rollout_corr/log_ppl_diff_min:-0.15778684616088867 - rollout_corr/ppl_ratio:1.0023249387741089 - rollout_corr/chi2_token:0.0031723976135253906 - rollout_corr/chi2_seq:0.40077733993530273 - actor/pg_clipfrac:0.0006346500516277349 - actor/ppo_kl:-0.00015972995151969371 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.001086011267034337 - actor/loss:-0.0010860112961381674 - actor/grad_norm:0.21710205078125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.143773555755615 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:11 - training/epoch:0 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.002921799663454294 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.002921799663454294 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:90.9384765625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.478515625 - response_length_non_aborted/mean:90.9384765625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.478515625 - response/aborted_ratio:0.0 - prompt_length/mean:60.953125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.136928942003578 - timing_s/old_log_prob:2.5185590160035645 - timing_s/adv:0.18633528200007277 - timing_s/update_actor:5.033867088997795 - timing_s/update_weights:0.9832612669997616 - timing_s/step:13.055665241001407 - timing_per_token_ms/adv:0.0011980125757862937 - timing_per_token_ms/update_actor:0.03236443475827484 - timing_per_token_ms/gen:0.04442530623601098 - perf/total_num_tokens:155537 - perf/time_per_step:13.055665241001407 - perf/throughput:11913.372250962362 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 3%|▎ | 11/350 [02:35<1:20:29, 14.25s/it] (TaskRunnerV1 pid=2350) Training Progress: 3%|▎ | 12/350 [02:48<1:18:08, 13.87s/it] (TaskRunnerV1 pid=2350) Training Progress: 4%|▎ | 13/350 [03:01<1:16:53, 13.69s/it] (TaskRunnerV1 pid=2350) step:12 - global_seqlen/min:157461.0 - global_seqlen/max:157461.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157461.0 - global_seqlen/balanced_max:157461.0 - global_seqlen/mean:157461.0 - actor/entropy:1.6641335487365723 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.33283302187919617 - training/rollout_probs_diff_mean:0.006899131461977959 - training/rollout_probs_diff_std:0.01139399129897356 - training/rollout_actor_probs_pearson_corr:0.9994345307350159 - rollout_corr/training_ppl:10.226015090942383 - rollout_corr/training_log_ppl:1.6928455829620361 - rollout_corr/kl:0.0014989308547228575 - rollout_corr/k3_kl:0.0016441144980490208 - rollout_corr/rollout_ppl:10.198142051696777 - rollout_corr/rollout_log_ppl:1.691562294960022 - rollout_corr/log_ppl_diff:0.0012834048829972744 - rollout_corr/log_ppl_abs_diff:0.006379385478794575 - rollout_corr/log_ppl_diff_max:0.06445908546447754 - rollout_corr/log_ppl_diff_min:-0.10207509994506836 - rollout_corr/ppl_ratio:1.0013329982757568 - rollout_corr/chi2_token:0.0035866498947143555 - rollout_corr/chi2_seq:0.3700447082519531 - actor/pg_clipfrac:0.0008499487121298444 - actor/ppo_kl:0.00011607123057425875 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0020448998839128762 - actor/loss:0.0020448995055630803 - actor/grad_norm:0.249267578125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.136566638946533 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:12 - training/epoch:0 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00027978981961496174 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:-0.00027978981961496174 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:91.6142578125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4921875 - response_length_non_aborted/mean:91.6142578125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4921875 - response/aborted_ratio:0.0 - prompt_length/mean:62.15625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.137957177998032 - timing_s/old_log_prob:2.564958051996655 - timing_s/adv:0.18913939600315643 - timing_s/update_actor:4.843610483003431 - timing_s/update_weights:0.9778468759977841 - timing_s/step:12.885454767005285 - timing_per_token_ms/adv:0.0012011824896523992 - timing_per_token_ms/update_actor:0.03076069936684913 - timing_per_token_ms/gen:0.04410856894031778 - perf/total_num_tokens:157461 - perf/time_per_step:12.885454767005285 - perf/throughput:12220.057642295817 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:13 - global_seqlen/min:160327.0 - global_seqlen/max:160327.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160327.0 - global_seqlen/balanced_max:160327.0 - global_seqlen/mean:160327.0 - actor/entropy:1.656590461730957 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2879513204097748 - training/rollout_probs_diff_mean:0.006717274431139231 - training/rollout_probs_diff_std:0.011081576347351074 - training/rollout_actor_probs_pearson_corr:0.9994685053825378 - rollout_corr/training_ppl:8.970175743103027 - rollout_corr/training_log_ppl:1.6998234987258911 - rollout_corr/kl:0.0013920244527980685 - rollout_corr/k3_kl:0.0016250383341684937 - rollout_corr/rollout_ppl:8.946610450744629 - rollout_corr/rollout_log_ppl:1.698757290840149 - rollout_corr/log_ppl_diff:0.0010662738932296634 - rollout_corr/log_ppl_abs_diff:0.005766352638602257 - rollout_corr/log_ppl_diff_max:0.04379546642303467 - rollout_corr/log_ppl_diff_min:-0.08659601211547852 - rollout_corr/ppl_ratio:1.0011032819747925 - rollout_corr/chi2_token:0.0037431716918945312 - rollout_corr/chi2_seq:0.4625610113143921 - actor/pg_clipfrac:0.0010379690162759894 - actor/ppo_kl:0.0006067637132976719 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0017413290770491585 - actor/loss:0.0017413293826393783 - actor/grad_norm:0.2589111328125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.09093189239502 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:13 - training/epoch:0 - critic/score/mean:0.03515625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.03515625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0032340679317712784 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:-0.0032340679317712784 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.0849609375 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.50390625 - response_length_non_aborted/mean:94.0849609375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.50390625 - response/aborted_ratio:0.0 - prompt_length/mean:62.484375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.146603382003377 - timing_s/old_log_prob:2.4526613909984007 - timing_s/adv:0.4174553749980987 - timing_s/update_actor:4.894213751998905 - timing_s/update_weights:0.9947760200011544 - timing_s/step:13.089039783000771 - timing_per_token_ms/adv:0.0026037746293394045 - timing_per_token_ms/update_actor:0.030526447522868292 - timing_per_token_ms/gen:0.043040006871317864 - perf/total_num_tokens:160327 - perf/time_per_step:13.089039783000771 - perf/throughput:12248.95046985973 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:14 - global_seqlen/min:156788.0 - global_seqlen/max:156788.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156788.0 - global_seqlen/balanced_max:156788.0 - global_seqlen/mean:156788.0 - actor/entropy:1.728095293045044 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3058863878250122 - training/rollout_probs_diff_mean:0.006770497653633356 - training/rollout_probs_diff_std:0.01108667254447937 - training/rollout_actor_probs_pearson_corr:0.9994663596153259 - rollout_corr/training_ppl:10.167875289916992 - rollout_corr/training_log_ppl:1.7728216648101807 - rollout_corr/kl:0.0018742319662123919 - rollout_corr/k3_kl:0.0017049115849658847 - rollout_corr/rollout_ppl:10.10560131072998 - rollout_corr/rollout_log_ppl:1.7703906297683716 - rollout_corr/log_ppl_diff:0.0024310334119945765 - rollout_corr/log_ppl_abs_diff:0.0063042594119906425 - rollout_corr/log_ppl_diff_max:0.1531233787536621 - rollout_corr/log_ppl_diff_min:-0.032785773277282715 - rollout_corr/ppl_ratio:1.0024887323379517 - rollout_corr/chi2_token:0.003070235252380371 - rollout_corr/chi2_seq:0.33868932723999023 - actor/pg_clipfrac:0.0005471983380023934 - actor/ppo_kl:-7.161754971699945e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.005480019928654656 - actor/loss:0.005480019433889538 - actor/grad_norm:0.1917724609375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.192558765411377 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:14 - training/epoch:0 - critic/score/mean:0.0244140625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0244140625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00221485598012805 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.00221485598012805 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.47265625 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4775390625 - response_length_non_aborted/mean:91.47265625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4775390625 - response/aborted_ratio:0.0 - prompt_length/mean:61.640625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.143066855001962 - timing_s/old_log_prob:2.345719997996639 - timing_s/adv:0.18919314500089968 - timing_s/update_actor:5.041598048002925 - timing_s/update_weights:0.9895776339981239 - timing_s/step:12.893151263000618 - timing_per_token_ms/adv:0.0012066812830120908 - timing_per_token_ms/update_actor:0.03215550965636991 - timing_per_token_ms/gen:0.04423140085196611 - perf/total_num_tokens:156788 - perf/time_per_step:12.893151263000618 - perf/throughput:12160.564690645751 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 4%|▍ | 14/350 [03:14<1:15:36, 13.50s/it] (TaskRunnerV1 pid=2350) Training Progress: 4%|▍ | 15/350 [03:32<1:22:47, 14.83s/it] (TaskRunnerV1 pid=2350) Training Progress: 5%|▍ | 16/350 [03:45<1:19:30, 14.28s/it] (TransferQueueController pid=2570) 2026-11-01 16:54:05,013 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_ebda4368: [Performance] Total success requests: 89362, Total req/min: 17701.57, Total avg process time: 0.0001s; (TransferQueueController pid=2570) Time range: last 5.05 minutes; (TransferQueueController pid=2570) Per-operation statistics: KV_RETRIEVE_META: req_count=33949, req/min=6724.90, avg_time=0.000250s, max_time=0.067784s, min_time=0.000028s; SET_CUSTOM_META: req_count=33832, req/min=6701.72, avg_time=0.000010s, max_time=0.000720s, min_time=0.000003s; KV_LIST: req_count=64, req/min=12.68, avg_time=0.000301s, max_time=0.000865s, min_time=0.000114s; NOTIFY_DATA_UPDATE: req_count=21459, req/min=4250.78, avg_time=0.000086s, max_time=0.003325s, min_time=0.000030s; CLEAR_META: req_count=41, req/min=8.12, avg_time=0.001605s, max_time=0.003777s, min_time=0.000802s; KV_RETRIEVE_KEYS: req_count=17, req/min=3.37, avg_time=0.000122s, max_time=0.000218s, min_time=0.000101s (SimpleStorageUnit pid=2676) Per-operation statistics: PUT_DATA: req_count=2729, req/min=543.66, avg_time=0.000611s, max_time=0.012798s, min_time=0.000313s; CLEAR_DATA: req_count=41, req/min=8.17, avg_time=0.001378s, max_time=0.004056s, min_time=0.000379s; GET_DATA: req_count=93, req/min=18.53, avg_time=0.000585s, max_time=0.001252s, min_time=0.000375s (TaskRunnerV1 pid=2350) step:15 - global_seqlen/min:159672.0 - global_seqlen/max:159672.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159672.0 - global_seqlen/balanced_max:159672.0 - global_seqlen/mean:159672.0 - actor/entropy:1.6431738138198853 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.21887686848640442 - training/rollout_probs_diff_mean:0.006846655160188675 - training/rollout_probs_diff_std:0.011289326474070549 - training/rollout_actor_probs_pearson_corr:0.9994475841522217 - rollout_corr/training_ppl:9.639467239379883 - rollout_corr/training_log_ppl:1.714531660079956 - rollout_corr/kl:0.0012641774956136942 - rollout_corr/k3_kl:0.00166707718744874 - rollout_corr/rollout_ppl:9.610626220703125 - rollout_corr/rollout_log_ppl:1.7128684520721436 - rollout_corr/log_ppl_diff:0.0016629797173663974 - rollout_corr/log_ppl_abs_diff:0.006106236949563026 - rollout_corr/log_ppl_diff_max:0.08859694004058838 - rollout_corr/log_ppl_diff_min:-0.07607507705688477 - rollout_corr/ppl_ratio:1.001711130142212 - rollout_corr/chi2_token:0.004189491271972656 - rollout_corr/chi2_seq:0.4277505874633789 - actor/pg_clipfrac:0.000923716892430093 - actor/ppo_kl:0.00019347612154282007 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.006111636379500851 - actor/loss:-0.006111635564593598 - actor/grad_norm:0.265869140625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.310100078582764 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.039) - val-core/char_count/acc/mean@1:np.float64(0.039) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:15 - training/epoch:0 - critic/score/mean:0.0341796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0341796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.012193755246698856 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.012193755246698856 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.7734375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5078125 - response_length_non_aborted/mean:92.7734375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5078125 - response/aborted_ratio:0.0 - prompt_length/mean:63.15625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.147460578999016 - timing_s/old_log_prob:2.3294003829942085 - timing_s/adv:0.18437678799818968 - timing_s/update_actor:4.8551880739978515 - timing_s/update_weights:0.9873432599997614 - timing_s/step:12.692728365997027 - timing_s/testing:5.073971357000119 - timing_per_token_ms/adv:0.00115472210530456 - timing_per_token_ms/update_actor:0.030407260346196275 - timing_per_token_ms/gen:0.04365747977893702 - perf/total_num_tokens:159672 - perf/time_per_step:12.692728365997027 - perf/throughput:12579.80123704141 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:16 - global_seqlen/min:156144.0 - global_seqlen/max:156144.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156144.0 - global_seqlen/balanced_max:156144.0 - global_seqlen/mean:156144.0 - actor/entropy:1.687233328819275 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2892291247844696 - training/rollout_probs_diff_mean:0.0068573965691030025 - training/rollout_probs_diff_std:0.011236241087317467 - training/rollout_actor_probs_pearson_corr:0.9994503855705261 - rollout_corr/training_ppl:9.194076538085938 - rollout_corr/training_log_ppl:1.7461564540863037 - rollout_corr/kl:0.0015515421982854605 - rollout_corr/k3_kl:0.0017050220631062984 - rollout_corr/rollout_ppl:9.170737266540527 - rollout_corr/rollout_log_ppl:1.744568109512329 - rollout_corr/log_ppl_diff:0.0015881758881732821 - rollout_corr/log_ppl_abs_diff:0.006034161429852247 - rollout_corr/log_ppl_diff_max:0.12365245819091797 - rollout_corr/log_ppl_diff_min:-0.049256324768066406 - rollout_corr/ppl_ratio:1.0016345977783203 - rollout_corr/chi2_token:0.0037469863891601562 - rollout_corr/chi2_seq:0.3616023063659668 - actor/pg_clipfrac:0.0005465156949261049 - actor/ppo_kl:9.567913442272888e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0010555746775935404 - actor/loss:0.001055574044585228 - actor/grad_norm:0.2225341796875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.390624046325684 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:16 - training/epoch:0 - critic/score/mean:0.0244140625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0244140625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0029347622767090797 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0029347622767090797 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.25 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.490234375 - response_length_non_aborted/mean:91.25 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.490234375 - response/aborted_ratio:0.0 - prompt_length/mean:61.234375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.155926637999073 - timing_s/old_log_prob:2.353683387002093 - timing_s/adv:0.3813405769978999 - timing_s/update_actor:4.8294427639993955 - timing_s/update_weights:0.9942434559998219 - timing_s/step:12.884729145996971 - timing_per_token_ms/adv:0.0024422365060322517 - timing_per_token_ms/update_actor:0.030929416205549978 - timing_per_token_ms/gen:0.04447695460187364 - perf/total_num_tokens:156144 - perf/time_per_step:12.884729145996971 - perf/throughput:12118.531808525508 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:17 - global_seqlen/min:156059.0 - global_seqlen/max:156059.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156059.0 - global_seqlen/balanced_max:156059.0 - global_seqlen/mean:156059.0 - actor/entropy:1.6732383966445923 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.22637814283370972 - training/rollout_probs_diff_mean:0.006742248777300119 - training/rollout_probs_diff_std:0.01101083867251873 - training/rollout_actor_probs_pearson_corr:0.999474287033081 - rollout_corr/training_ppl:10.084066390991211 - rollout_corr/training_log_ppl:1.7392358779907227 - rollout_corr/kl:0.0013765380717813969 - rollout_corr/k3_kl:0.0016361494781449437 - rollout_corr/rollout_ppl:10.056217193603516 - rollout_corr/rollout_log_ppl:1.7378690242767334 - rollout_corr/log_ppl_diff:0.0013668875908479095 - rollout_corr/log_ppl_abs_diff:0.0063904086127877235 - rollout_corr/log_ppl_diff_max:0.1486802101135254 - rollout_corr/log_ppl_diff_min:-0.1806039810180664 - rollout_corr/ppl_ratio:1.0014334917068481 - rollout_corr/chi2_token:0.00380551815032959 - rollout_corr/chi2_seq:0.41309428215026855 - actor/pg_clipfrac:0.0006679254759850057 - actor/ppo_kl:0.0002618622220325051 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.015435263703693636 - actor/loss:-0.01543526363093406 - actor/grad_norm:0.2369384765625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.412205696105957 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:17 - training/epoch:0 - critic/score/mean:0.029296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.029296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.015613156370818615 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.015613156370818615 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.0888671875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.484375 - response_length_non_aborted/mean:91.0888671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.484375 - response/aborted_ratio:0.0 - prompt_length/mean:61.3125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0234375 - timing_s/gen:4.130009619999328 - timing_s/old_log_prob:2.3704623020021245 - timing_s/adv:0.19689782299974468 - timing_s/update_actor:5.064559475998976 - timing_s/update_weights:1.004891807999229 - timing_s/step:12.940264953998849 - timing_per_token_ms/adv:0.001261688355043571 - timing_per_token_ms/update_actor:0.03245285101146987 - timing_per_token_ms/gen:0.04427777668184752 - perf/total_num_tokens:156059 - perf/time_per_step:12.940264953998849 - perf/throughput:12059.953992810175 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 5%|▍ | 17/350 [03:58<1:17:16, 13.92s/it] (TaskRunnerV1 pid=2350) Training Progress: 5%|▌ | 18/350 [04:11<1:15:32, 13.65s/it] (SimpleStorageUnit pid=2691) 2026-11-01 16:54:05,027 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_52739293: [Performance] Total success requests: 2840, Total req/min: 565.81, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2691) Time range: last 5.02 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2691) Per-operation statistics: PUT_DATA: req_count=2706, req/min=539.12, avg_time=0.000618s, max_time=0.005474s, min_time=0.000315s; CLEAR_DATA: req_count=41, req/min=8.17, avg_time=0.001731s, max_time=0.004487s, min_time=0.000373s; GET_DATA: req_count=93, req/min=18.53, avg_time=0.000664s, max_time=0.001249s, min_time=0.000400s [repeated 7x across cluster] (TaskRunnerV1 pid=2350) Training Progress: 5%|▌ | 19/350 [04:24<1:13:49, 13.38s/it] (TaskRunnerV1 pid=2350) step:18 - global_seqlen/min:159588.0 - global_seqlen/max:159588.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159588.0 - global_seqlen/balanced_max:159588.0 - global_seqlen/mean:159588.0 - actor/entropy:1.6888456344604492 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.29472845792770386 - training/rollout_probs_diff_mean:0.006890475749969482 - training/rollout_probs_diff_std:0.01149087492376566 - training/rollout_actor_probs_pearson_corr:0.9994305968284607 - rollout_corr/training_ppl:9.465553283691406 - rollout_corr/training_log_ppl:1.7434203624725342 - rollout_corr/kl:0.0017573360819369555 - rollout_corr/k3_kl:0.0017101338598877192 - rollout_corr/rollout_ppl:9.412256240844727 - rollout_corr/rollout_log_ppl:1.7413322925567627 - rollout_corr/log_ppl_diff:0.002087949076667428 - rollout_corr/log_ppl_abs_diff:0.006241351366043091 - rollout_corr/log_ppl_diff_max:0.09681892395019531 - rollout_corr/log_ppl_diff_min:-0.057350993156433105 - rollout_corr/ppl_ratio:1.0021339654922485 - rollout_corr/chi2_token:0.0033376216888427734 - rollout_corr/chi2_seq:0.3335838317871094 - actor/pg_clipfrac:0.0007892886163366104 - actor/ppo_kl:9.387186014464534e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0017935175419552252 - actor/loss:-0.0017935173818841577 - actor/grad_norm:0.242431640625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.458524703979492 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:18 - training/epoch:0 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0026634456589818 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.0026634456589818 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.34765625 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.46875 - response_length_non_aborted/mean:93.34765625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.46875 - response/aborted_ratio:0.0 - prompt_length/mean:62.5 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.149718010005017 - timing_s/old_log_prob:2.3428765360004036 - timing_s/adv:0.1825383530012914 - timing_s/update_actor:4.849021553003695 - timing_s/update_weights:0.979519202002848 - timing_s/step:12.698362745999475 - timing_per_token_ms/adv:0.0011438100170519801 - timing_per_token_ms/update_actor:0.030384625115946657 - timing_per_token_ms/gen:0.04341254142784677 - perf/total_num_tokens:159588 - perf/time_per_step:12.698362745999475 - perf/throughput:12567.604437845896 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:19 - global_seqlen/min:159551.0 - global_seqlen/max:159551.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159551.0 - global_seqlen/balanced_max:159551.0 - global_seqlen/mean:159551.0 - actor/entropy:1.6529279947280884 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.29952651262283325 - training/rollout_probs_diff_mean:0.0069153327494859695 - training/rollout_probs_diff_std:0.011509661562740803 - training/rollout_actor_probs_pearson_corr:0.9994252324104309 - rollout_corr/training_ppl:9.473451614379883 - rollout_corr/training_log_ppl:1.6977579593658447 - rollout_corr/kl:0.0017692343099042773 - rollout_corr/k3_kl:0.0016716532409191132 - rollout_corr/rollout_ppl:9.494902610778809 - rollout_corr/rollout_log_ppl:1.696017861366272 - rollout_corr/log_ppl_diff:0.0017402844969183207 - rollout_corr/log_ppl_abs_diff:0.006039258558303118 - rollout_corr/log_ppl_diff_max:0.060004234313964844 - rollout_corr/log_ppl_diff_min:-0.0840463638305664 - rollout_corr/ppl_ratio:1.001780390739441 - rollout_corr/chi2_token:0.0031626224517822266 - rollout_corr/chi2_seq:0.5505149364471436 - actor/pg_clipfrac:0.0008592399335611846 - actor/ppo_kl:-0.00020460436159434417 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.004480213756323792 - actor/loss:-0.004480213392525911 - actor/grad_norm:0.2774658203125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.42351484298706 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:19 - training/epoch:0 - critic/score/mean:0.0400390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0400390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.007309831213206053 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.007309831213206053 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.5771484375 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.48828125 - response_length_non_aborted/mean:93.5771484375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.48828125 - response/aborted_ratio:0.0 - prompt_length/mean:62.234375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.145903119999275 - timing_s/old_log_prob:2.365966147001018 - timing_s/adv:0.18230571699677967 - timing_s/update_actor:4.756665349996183 - timing_s/update_weights:0.9869847039954038 - timing_s/step:12.618867232995399 - timing_per_token_ms/adv:0.0011426172007494761 - timing_per_token_ms/update_actor:0.029812820665468615 - timing_per_token_ms/gen:0.04326626300574262 - perf/total_num_tokens:159551 - perf/time_per_step:12.618867232995399 - perf/throughput:12643.844891466271 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:20 - global_seqlen/min:154533.0 - global_seqlen/max:154533.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:154533.0 - global_seqlen/balanced_max:154533.0 - global_seqlen/mean:154533.0 - actor/entropy:1.68897545337677 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2802017033100128 - training/rollout_probs_diff_mean:0.006781827658414841 - training/rollout_probs_diff_std:0.011260956525802612 - training/rollout_actor_probs_pearson_corr:0.9994555115699768 - rollout_corr/training_ppl:10.670597076416016 - rollout_corr/training_log_ppl:1.7573497295379639 - rollout_corr/kl:0.0016191730974242091 - rollout_corr/k3_kl:0.0016947626136243343 - rollout_corr/rollout_ppl:10.664421081542969 - rollout_corr/rollout_log_ppl:1.7555968761444092 - rollout_corr/log_ppl_diff:0.0017528278985992074 - rollout_corr/log_ppl_abs_diff:0.0068343947641551495 - rollout_corr/log_ppl_diff_max:0.069122314453125 - rollout_corr/log_ppl_diff_min:-0.11672258377075195 - rollout_corr/ppl_ratio:1.001815915107727 - rollout_corr/chi2_token:0.003551959991455078 - rollout_corr/chi2_seq:0.4458174705505371 - actor/pg_clipfrac:0.000873923113734539 - actor/ppo_kl:0.0001247816926479572 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.00428743598604342 - actor/loss:-0.004287436662707478 - actor/grad_norm:0.2486572265625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.48476791381836 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.03) - val-core/char_count/acc/mean@1:np.float64(0.03) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:20 - training/epoch:0 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.005896789021790028 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.005896789021790028 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:89.7080078125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4599609375 - response_length_non_aborted/mean:89.7080078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4599609375 - response/aborted_ratio:0.0 - prompt_length/mean:61.203125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.13026703200012 - timing_s/old_log_prob:2.4374919410038274 - timing_s/adv:0.18203223800082924 - timing_s/update_actor:4.831931939996139 - timing_s/update_weights:1.0108777939967695 - timing_s/step:12.763394536996202 - timing_s/testing:4.929404004004027 - timing_per_token_ms/adv:0.0011779505866114633 - timing_per_token_ms/update_actor:0.031267961794543164 - timing_per_token_ms/gen:0.04496213879666148 - perf/total_num_tokens:154533 - perf/time_per_step:12.763394536996202 - perf/throughput:12107.515720215959 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 6%|▌ | 20/350 [04:42<1:20:55, 14.71s/it] (TaskRunnerV1 pid=2350) Training Progress: 6%|▌ | 21/350 [04:55<1:18:02, 14.23s/it] (TaskRunnerV1 pid=2350) Training Progress: 6%|▋ | 22/350 [05:08<1:16:00, 13.90s/it] (TaskRunnerV1 pid=2350) step:21 - global_seqlen/min:159743.0 - global_seqlen/max:159743.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159743.0 - global_seqlen/balanced_max:159743.0 - global_seqlen/mean:159743.0 - actor/entropy:1.7186092138290405 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3944442868232727 - training/rollout_probs_diff_mean:0.0068291486240923405 - training/rollout_probs_diff_std:0.011335560120642185 - training/rollout_actor_probs_pearson_corr:0.9994462132453918 - rollout_corr/training_ppl:10.069716453552246 - rollout_corr/training_log_ppl:1.751887321472168 - rollout_corr/kl:0.0020545076113194227 - rollout_corr/k3_kl:0.0017155472887679935 - rollout_corr/rollout_ppl:10.03875732421875 - rollout_corr/rollout_log_ppl:1.7500498294830322 - rollout_corr/log_ppl_diff:0.0018376868683844805 - rollout_corr/log_ppl_abs_diff:0.005974411033093929 - rollout_corr/log_ppl_diff_max:0.05300557613372803 - rollout_corr/log_ppl_diff_min:-0.0664362907409668 - rollout_corr/ppl_ratio:1.001875638961792 - rollout_corr/chi2_token:0.0027898550033569336 - rollout_corr/chi2_seq:0.23130154609680176 - actor/pg_clipfrac:0.0009775418936139129 - actor/ppo_kl:-0.00023015382672989893 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0017371819631080143 - actor/loss:-0.0017371817957609892 - actor/grad_norm:0.2529296875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.537919521331787 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:21 - training/epoch:0 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0024747515562921762 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0024747515562921762 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.1708984375 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.498046875 - response_length_non_aborted/mean:94.1708984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.498046875 - response/aborted_ratio:0.0 - prompt_length/mean:61.828125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.131585216993699 - timing_s/old_log_prob:2.3818461379996734 - timing_s/adv:0.20181389000208583 - timing_s/update_actor:5.0630329819978215 - timing_s/update_weights:0.9965917219960829 - timing_s/step:12.945470754995767 - timing_per_token_ms/adv:0.001263366094302009 - timing_per_token_ms/update_actor:0.03169486601602462 - timing_per_token_ms/gen:0.04284498985796786 - perf/total_num_tokens:159743 - perf/time_per_step:12.945470754995767 - perf/throughput:12339.682582678874 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:22 - global_seqlen/min:159134.0 - global_seqlen/max:159134.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159134.0 - global_seqlen/balanced_max:159134.0 - global_seqlen/mean:159134.0 - actor/entropy:1.6452534198760986 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.21391791105270386 - training/rollout_probs_diff_mean:0.0068138595670461655 - training/rollout_probs_diff_std:0.01127750426530838 - training/rollout_actor_probs_pearson_corr:0.9994507431983948 - rollout_corr/training_ppl:8.46743106842041 - rollout_corr/training_log_ppl:1.6961904764175415 - rollout_corr/kl:0.0016619691159576178 - rollout_corr/k3_kl:0.0016408298397436738 - rollout_corr/rollout_ppl:8.45666790008545 - rollout_corr/rollout_log_ppl:1.6945072412490845 - rollout_corr/log_ppl_diff:0.001683276379480958 - rollout_corr/log_ppl_abs_diff:0.005761025473475456 - rollout_corr/log_ppl_diff_max:0.03679466247558594 - rollout_corr/log_ppl_diff_min:-0.0501328706741333 - rollout_corr/ppl_ratio:1.0017149448394775 - rollout_corr/chi2_token:0.0032650232315063477 - rollout_corr/chi2_seq:0.4059886932373047 - actor/pg_clipfrac:0.0007594125333222715 - actor/ppo_kl:-0.0002130858800531617 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.00039752500379108824 - actor/loss:-0.00039752511656843126 - actor/grad_norm:0.2231292724609375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.601224422454834 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:22 - training/epoch:0 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0016711230855435133 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.0016711230855435133 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.794921875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.4951171875 - response_length_non_aborted/mean:93.794921875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.4951171875 - response/aborted_ratio:0.0 - prompt_length/mean:61.609375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.141947033996985 - timing_s/old_log_prob:2.4329856480035232 - timing_s/adv:0.2462606639965088 - timing_s/update_actor:4.832836399997177 - timing_s/update_weights:0.9991244859993458 - timing_s/step:12.826324828005454 - timing_per_token_ms/adv:0.001547505020903822 - timing_per_token_ms/update_actor:0.030369602976090447 - timing_per_token_ms/gen:0.043124617724808786 - perf/total_num_tokens:159134 - perf/time_per_step:12.826324828005454 - perf/throughput:12406.827531183459 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:23 - global_seqlen/min:159211.0 - global_seqlen/max:159211.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159211.0 - global_seqlen/balanced_max:159211.0 - global_seqlen/mean:159211.0 - actor/entropy:1.687877893447876 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2895875573158264 - training/rollout_probs_diff_mean:0.006858129054307938 - training/rollout_probs_diff_std:0.011416172608733177 - training/rollout_actor_probs_pearson_corr:0.9994413256645203 - rollout_corr/training_ppl:9.21562671661377 - rollout_corr/training_log_ppl:1.725479006767273 - rollout_corr/kl:0.0017310639377683401 - rollout_corr/k3_kl:0.0017010491574183106 - rollout_corr/rollout_ppl:9.182461738586426 - rollout_corr/rollout_log_ppl:1.7229506969451904 - rollout_corr/log_ppl_diff:0.0025284220464527607 - rollout_corr/log_ppl_abs_diff:0.006274046376347542 - rollout_corr/log_ppl_diff_max:0.09852302074432373 - rollout_corr/log_ppl_diff_min:-0.0460052490234375 - rollout_corr/ppl_ratio:1.0025749206542969 - rollout_corr/chi2_token:0.003359198570251465 - rollout_corr/chi2_seq:0.4013509750366211 - actor/pg_clipfrac:0.0006469044437835691 - actor/ppo_kl:3.7643517619774996e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0004248176483088173 - actor/loss:0.00042481772834435105 - actor/grad_norm:0.21917724609375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.666197776794434 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:23 - training/epoch:0 - critic/score/mean:0.0205078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0205078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0005107159377075732 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.0005107159377075732 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.1826171875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.5087890625 - response_length_non_aborted/mean:93.1826171875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.5087890625 - response/aborted_ratio:0.0 - prompt_length/mean:62.296875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.1550256240007 - timing_s/old_log_prob:2.387989141003345 - timing_s/adv:0.18292676899727667 - timing_s/update_actor:4.872246786006144 - timing_s/update_weights:0.9916496709993226 - timing_s/step:12.765919527999358 - timing_per_token_ms/adv:0.0011489581058926623 - timing_per_token_ms/update_actor:0.03060245074778843 - timing_per_token_ms/gen:0.04354505521961769 - perf/total_num_tokens:159211 - perf/time_per_step:12.765919527999358 - perf/throughput:12471.565377707746 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 7%|▋ | 23/350 [05:21<1:14:07, 13.60s/it] (TaskRunnerV1 pid=2350) Training Progress: 7%|▋ | 24/350 [05:34<1:12:49, 13.40s/it] (TaskRunnerV1 pid=2350) Training Progress: 7%|▋ | 25/350 [05:52<1:20:10, 14.80s/it] (TaskRunnerV1 pid=2350) step:24 - global_seqlen/min:159006.0 - global_seqlen/max:159006.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159006.0 - global_seqlen/balanced_max:159006.0 - global_seqlen/mean:159006.0 - actor/entropy:1.7135592699050903 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.30051079392433167 - training/rollout_probs_diff_mean:0.006934736855328083 - training/rollout_probs_diff_std:0.011594129726290703 - training/rollout_actor_probs_pearson_corr:0.9994235634803772 - rollout_corr/training_ppl:12.709484100341797 - rollout_corr/training_log_ppl:1.779989242553711 - rollout_corr/kl:0.002037273719906807 - rollout_corr/k3_kl:0.0017429892905056477 - rollout_corr/rollout_ppl:12.639788627624512 - rollout_corr/rollout_log_ppl:1.777514100074768 - rollout_corr/log_ppl_diff:0.0024749988224357367 - rollout_corr/log_ppl_abs_diff:0.006203006487339735 - rollout_corr/log_ppl_diff_max:0.07121467590332031 - rollout_corr/log_ppl_diff_min:-0.045757293701171875 - rollout_corr/ppl_ratio:1.002521276473999 - rollout_corr/chi2_token:0.002885103225708008 - rollout_corr/chi2_seq:0.2123410701751709 - actor/pg_clipfrac:0.0009916840481309919 - actor/ppo_kl:0.00012802788198011209 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.003948692392441444 - actor/loss:0.0039486919122282416 - actor/grad_norm:0.23321533203125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.72281789779663 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:24 - training/epoch:0 - critic/score/mean:0.0263671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0263671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00021151514374651015 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.00021151514374651015 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.388671875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5 - response_length_non_aborted/mean:93.388671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5 - response/aborted_ratio:0.0 - prompt_length/mean:61.890625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.1285368769968045 - timing_s/old_log_prob:2.4815750880006817 - timing_s/adv:0.18382264299725648 - timing_s/update_actor:4.859289418003755 - timing_s/update_weights:0.9941167540018796 - timing_s/step:12.825065078002808 - timing_per_token_ms/adv:0.001156073626135218 - timing_per_token_ms/update_actor:0.030560415443466 - timing_per_token_ms/gen:0.04317198449228071 - perf/total_num_tokens:159006 - perf/time_per_step:12.825065078002808 - perf/throughput:12398.065743363955 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:25 - global_seqlen/min:157608.0 - global_seqlen/max:157608.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157608.0 - global_seqlen/balanced_max:157608.0 - global_seqlen/mean:157608.0 - actor/entropy:1.6878206729888916 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.22871974110603333 - training/rollout_probs_diff_mean:0.006656230427324772 - training/rollout_probs_diff_std:0.01105966605246067 - training/rollout_actor_probs_pearson_corr:0.9994782209396362 - rollout_corr/training_ppl:10.849997520446777 - rollout_corr/training_log_ppl:1.7512346506118774 - rollout_corr/kl:0.0018986215582117438 - rollout_corr/k3_kl:0.001622362993657589 - rollout_corr/rollout_ppl:10.826286315917969 - rollout_corr/rollout_log_ppl:1.7489970922470093 - rollout_corr/log_ppl_diff:0.0022377707064151764 - rollout_corr/log_ppl_abs_diff:0.006402141414582729 - rollout_corr/log_ppl_diff_max:0.05945444107055664 - rollout_corr/log_ppl_diff_min:-0.07735729217529297 - rollout_corr/ppl_ratio:1.0022847652435303 - rollout_corr/chi2_token:0.0026890039443969727 - rollout_corr/chi2_seq:0.32276344299316406 - actor/pg_clipfrac:0.0007301104797079461 - actor/ppo_kl:-0.00017580254764955802 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0038249312347033992 - actor/loss:-0.0038249307253863662 - actor/grad_norm:0.2244873046875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.779468059539795 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.027) - val-core/char_count/acc/mean@1:np.float64(0.027) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:25 - training/epoch:0 - critic/score/mean:0.029296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.029296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.004350642208009958 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.004350642208009958 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.2265625 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.4951171875 - response_length_non_aborted/mean:92.2265625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.4951171875 - response/aborted_ratio:0.0 - prompt_length/mean:61.6875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:4.123531026998535 - timing_s/old_log_prob:2.562934761001088 - timing_s/adv:0.19336709000344854 - timing_s/update_actor:4.846025937993545 - timing_s/update_weights:0.9934913950055488 - timing_s/step:12.906707823000033 - timing_s/testing:5.036420229000214 - timing_per_token_ms/adv:0.0012268862621405547 - timing_per_token_ms/update_actor:0.030747334767229743 - timing_per_token_ms/gen:0.043662971484524936 - perf/total_num_tokens:157608 - perf/time_per_step:12.906707823000033 - perf/throughput:12211.324697312752 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:26 - global_seqlen/min:159036.0 - global_seqlen/max:159036.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159036.0 - global_seqlen/balanced_max:159036.0 - global_seqlen/mean:159036.0 - actor/entropy:1.6812337636947632 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.34012773633003235 - training/rollout_probs_diff_mean:0.006877640262246132 - training/rollout_probs_diff_std:0.01130044087767601 - training/rollout_actor_probs_pearson_corr:0.9994438886642456 - rollout_corr/training_ppl:9.341169357299805 - rollout_corr/training_log_ppl:1.7367041110992432 - rollout_corr/kl:0.001668682205490768 - rollout_corr/k3_kl:0.0016648019663989544 - rollout_corr/rollout_ppl:9.309035301208496 - rollout_corr/rollout_log_ppl:1.7349408864974976 - rollout_corr/log_ppl_diff:0.0017633617389947176 - rollout_corr/log_ppl_abs_diff:0.00618129363283515 - rollout_corr/log_ppl_diff_max:0.06667423248291016 - rollout_corr/log_ppl_diff_min:-0.06675839424133301 - rollout_corr/ppl_ratio:1.0018101930618286 - rollout_corr/chi2_token:0.003328084945678711 - rollout_corr/chi2_seq:0.3947179317474365 - actor/pg_clipfrac:0.0005853064588639729 - actor/ppo_kl:1.2220110235716675e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0006742428449797444 - actor/loss:0.0006742430850863457 - actor/grad_norm:0.20233154296875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.612566471099854 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:26 - training/epoch:0 - critic/score/mean:0.0224609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0224609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0025050004478543997 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.35355237126350403 - critic/returns/mean:0.0025050004478543997 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.35355237126350403 - response_length/mean:93.44921875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.4951171875 - response_length_non_aborted/mean:93.44921875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.4951171875 - response/aborted_ratio:0.0 - prompt_length/mean:61.859375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.131332985001791 - timing_s/old_log_prob:2.386109717997897 - timing_s/adv:0.19383802999800537 - timing_s/update_actor:4.869954604000668 - timing_s/update_weights:1.0076929780043429 - timing_s/step:12.762863306998042 - timing_per_token_ms/adv:0.001218831145136984 - timing_per_token_ms/update_actor:0.030621712090348526 - timing_per_token_ms/gen:0.04317323271539722 - perf/total_num_tokens:159036 - perf/time_per_step:12.762863306998042 - perf/throughput:12460.84018723279 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 7%|▋ | 26/350 [06:05<1:17:07, 14.28s/it] (TaskRunnerV1 pid=2350) Training Progress: 8%|▊ | 27/350 [06:18<1:14:26, 13.83s/it] (TaskRunnerV1 pid=2350) Training Progress: 8%|▊ | 28/350 [06:31<1:12:45, 13.56s/it] (TaskRunnerV1 pid=2350) step:27 - global_seqlen/min:156192.0 - global_seqlen/max:156192.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156192.0 - global_seqlen/balanced_max:156192.0 - global_seqlen/mean:156192.0 - actor/entropy:1.6720505952835083 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.4650409519672394 - training/rollout_probs_diff_mean:0.006810074672102928 - training/rollout_probs_diff_std:0.011456269770860672 - training/rollout_actor_probs_pearson_corr:0.9994379878044128 - rollout_corr/training_ppl:11.062763214111328 - rollout_corr/training_log_ppl:1.760514259338379 - rollout_corr/kl:0.0018504031468182802 - rollout_corr/k3_kl:0.001687434152700007 - rollout_corr/rollout_ppl:11.055253982543945 - rollout_corr/rollout_log_ppl:1.7584015130996704 - rollout_corr/log_ppl_diff:0.002112512942403555 - rollout_corr/log_ppl_abs_diff:0.0067128767259418964 - rollout_corr/log_ppl_diff_max:0.07089686393737793 - rollout_corr/log_ppl_diff_min:-0.05290699005126953 - rollout_corr/ppl_ratio:1.002166509628296 - rollout_corr/chi2_token:0.0030814409255981445 - rollout_corr/chi2_seq:0.3403968811035156 - actor/pg_clipfrac:0.0007366997148589387 - actor/ppo_kl:0.0002430809411938147 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.003657819441286847 - actor/loss:-0.00365781964501366 - actor/grad_norm:0.257568359375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.726315021514893 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:27 - training/epoch:0 - critic/score/mean:0.02734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.02734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0005075961234979331 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0005075961234979331 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.09375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4951171875 - response_length_non_aborted/mean:91.09375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4951171875 - response/aborted_ratio:0.0 - prompt_length/mean:61.4375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.151295877003577 - timing_s/old_log_prob:2.3666941009942093 - timing_s/adv:0.1967554399961955 - timing_s/update_actor:4.744310798996594 - timing_s/update_weights:1.0033104920003098 - timing_s/step:12.634992219995183 - timing_per_token_ms/adv:0.001259702417513032 - timing_per_token_ms/update_actor:0.030374864263192698 - timing_per_token_ms/gen:0.04450360073974675 - perf/total_num_tokens:156192 - perf/time_per_step:12.634992219995183 - perf/throughput:12361.85961023564 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:28 - global_seqlen/min:157202.0 - global_seqlen/max:157202.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157202.0 - global_seqlen/balanced_max:157202.0 - global_seqlen/mean:157202.0 - actor/entropy:1.6871525049209595 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2487974613904953 - training/rollout_probs_diff_mean:0.006778911221772432 - training/rollout_probs_diff_std:0.011335586197674274 - training/rollout_actor_probs_pearson_corr:0.9994484186172485 - rollout_corr/training_ppl:10.020844459533691 - rollout_corr/training_log_ppl:1.7490220069885254 - rollout_corr/kl:0.001880226656794548 - rollout_corr/k3_kl:0.0016724485903978348 - rollout_corr/rollout_ppl:10.011881828308105 - rollout_corr/rollout_log_ppl:1.7470933198928833 - rollout_corr/log_ppl_diff:0.001928640529513359 - rollout_corr/log_ppl_abs_diff:0.006372262258082628 - rollout_corr/log_ppl_diff_max:0.0650033950805664 - rollout_corr/log_ppl_diff_min:-0.09666824340820312 - rollout_corr/ppl_ratio:1.0019781589508057 - rollout_corr/chi2_token:0.0029584169387817383 - rollout_corr/chi2_seq:0.3102841377258301 - actor/pg_clipfrac:0.0008604802842455683 - actor/ppo_kl:3.212549988044581e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.003754415796720423 - actor/loss:-0.0037544163642451167 - actor/grad_norm:0.2530517578125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.767630577087402 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:28 - training/epoch:0 - critic/score/mean:0.037109375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.037109375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00543669518083334 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.00543669518083334 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:91.689453125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4833984375 - response_length_non_aborted/mean:91.689453125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4833984375 - response/aborted_ratio:0.0 - prompt_length/mean:61.828125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.124895240005571 - timing_s/old_log_prob:2.390293908996682 - timing_s/adv:0.24042674800148234 - timing_s/update_actor:4.8495746920016245 - timing_s/update_weights:0.9937182809953811 - timing_s/step:12.798541409996687 - timing_per_token_ms/adv:0.0015294127810173048 - timing_per_token_ms/update_actor:0.030849319296202493 - timing_per_token_ms/gen:0.04393327553526011 - perf/total_num_tokens:157202 - perf/time_per_step:12.798541409996687 - perf/throughput:12282.805904523826 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:29 - global_seqlen/min:155766.0 - global_seqlen/max:155766.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155766.0 - global_seqlen/balanced_max:155766.0 - global_seqlen/mean:155766.0 - actor/entropy:1.7115752696990967 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.20519882440567017 - training/rollout_probs_diff_mean:0.006806215737015009 - training/rollout_probs_diff_std:0.011084944941103458 - training/rollout_actor_probs_pearson_corr:0.9994604587554932 - rollout_corr/training_ppl:9.567676544189453 - rollout_corr/training_log_ppl:1.763885259628296 - rollout_corr/kl:0.0018538756994530559 - rollout_corr/k3_kl:0.0016640026587992907 - rollout_corr/rollout_ppl:9.607612609863281 - rollout_corr/rollout_log_ppl:1.7625104188919067 - rollout_corr/log_ppl_diff:0.0013747658813372254 - rollout_corr/log_ppl_abs_diff:0.006289053708314896 - rollout_corr/log_ppl_diff_max:0.04225564002990723 - rollout_corr/log_ppl_diff_min:-0.18606853485107422 - rollout_corr/ppl_ratio:1.0014301538467407 - rollout_corr/chi2_token:0.0029560327529907227 - rollout_corr/chi2_seq:0.31728243827819824 - actor/pg_clipfrac:0.0005444327465132421 - actor/ppo_kl:6.106092769186876e-07 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.003013936864590505 - actor/loss:0.0030139369191601872 - actor/grad_norm:0.18536376953125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.970826148986816 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:29 - training/epoch:0 - critic/score/mean:0.01953125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.01953125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00125507649499923 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.00125507649499923 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.474609375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4814453125 - response_length_non_aborted/mean:91.474609375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4814453125 - response/aborted_ratio:0.0 - prompt_length/mean:60.640625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.134065635000297 - timing_s/old_log_prob:2.5421490170047036 - timing_s/adv:0.18890845900023123 - timing_s/update_actor:4.797162441005639 - timing_s/update_weights:0.9990476699967985 - timing_s/step:12.84554615799425 - timing_per_token_ms/adv:0.0012127708164826165 - timing_per_token_ms/update_actor:0.030797237144214007 - timing_per_token_ms/gen:0.04413436142842209 - perf/total_num_tokens:155766 - perf/time_per_step:12.84554615799425 - perf/throughput:12126.070630563354 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 8%|▊ | 29/350 [06:44<1:11:35, 13.38s/it] (TaskRunnerV1 pid=2350) Training Progress: 9%|▊ | 30/350 [07:02<1:18:31, 14.72s/it] (TaskRunnerV1 pid=2350) Training Progress: 9%|▉ | 31/350 [07:14<1:15:15, 14.15s/it] (TaskRunnerV1 pid=2350) step:30 - global_seqlen/min:158515.0 - global_seqlen/max:158515.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158515.0 - global_seqlen/balanced_max:158515.0 - global_seqlen/mean:158515.0 - actor/entropy:1.6741338968276978 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.22621694207191467 - training/rollout_probs_diff_mean:0.006825032643973827 - training/rollout_probs_diff_std:0.011194875463843346 - training/rollout_actor_probs_pearson_corr:0.9994567632675171 - rollout_corr/training_ppl:10.89434814453125 - rollout_corr/training_log_ppl:1.7232868671417236 - rollout_corr/kl:0.0015659767668694258 - rollout_corr/k3_kl:0.0016838221345096827 - rollout_corr/rollout_ppl:10.885963439941406 - rollout_corr/rollout_log_ppl:1.7218440771102905 - rollout_corr/log_ppl_diff:0.0014429977163672447 - rollout_corr/log_ppl_abs_diff:0.0061674038879573345 - rollout_corr/log_ppl_diff_max:0.06540632247924805 - rollout_corr/log_ppl_diff_min:-0.06290960311889648 - rollout_corr/ppl_ratio:1.001484990119934 - rollout_corr/chi2_token:0.0036274194717407227 - rollout_corr/chi2_seq:0.3801959753036499 - actor/pg_clipfrac:0.0009475474253122229 - actor/ppo_kl:0.00022877513538333005 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.007188315299572423 - actor/loss:-0.007188315852545202 - actor/grad_norm:0.2412109375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.09537696838379 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.036) - val-core/char_count/acc/mean@1:np.float64(0.036) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:30 - training/epoch:0 - critic/score/mean:0.0263671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0263671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.009706296026706696 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.009706296026706696 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.5654296875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.4892578125 - response_length_non_aborted/mean:92.5654296875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.4892578125 - response/aborted_ratio:0.0 - prompt_length/mean:62.234375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.148382343002595 - timing_s/old_log_prob:2.5233748050013673 - timing_s/adv:0.18315728699963074 - timing_s/update_actor:4.83636339300574 - timing_s/update_weights:1.0107543240010273 - timing_s/step:12.874600808005198 - timing_s/testing:4.6449979890021496 - timing_per_token_ms/adv:0.001155457130237711 - timing_per_token_ms/update_actor:0.03051044628587667 - timing_per_token_ms/gen:0.04376530898754676 - perf/total_num_tokens:158515 - perf/time_per_step:12.874600808005198 - perf/throughput:12312.226403279097 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:31 - global_seqlen/min:160464.0 - global_seqlen/max:160464.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160464.0 - global_seqlen/balanced_max:160464.0 - global_seqlen/mean:160464.0 - actor/entropy:1.7058793306350708 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2914563715457916 - training/rollout_probs_diff_mean:0.0067863683216273785 - training/rollout_probs_diff_std:0.011258332058787346 - training/rollout_actor_probs_pearson_corr:0.9994545578956604 - rollout_corr/training_ppl:9.96035099029541 - rollout_corr/training_log_ppl:1.7486764192581177 - rollout_corr/kl:0.0016985309775918722 - rollout_corr/k3_kl:0.0016684288857504725 - rollout_corr/rollout_ppl:9.932247161865234 - rollout_corr/rollout_log_ppl:1.7468181848526 - rollout_corr/log_ppl_diff:0.0018581999465823174 - rollout_corr/log_ppl_abs_diff:0.0064966618083417416 - rollout_corr/log_ppl_diff_max:0.19312763214111328 - rollout_corr/log_ppl_diff_min:-0.07921552658081055 - rollout_corr/ppl_ratio:1.0019237995147705 - rollout_corr/chi2_token:0.003301262855529785 - rollout_corr/chi2_seq:0.3695485591888428 - actor/pg_clipfrac:0.0008674425274269499 - actor/ppo_kl:0.00012419245937659679 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0018793203926179558 - actor/loss:0.0018793204799294472 - actor/grad_norm:0.2393798828125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.15605592727661 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:31 - training/epoch:0 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00030032123322598636 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:-0.00030032123322598636 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.703125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5205078125 - response_length_non_aborted/mean:94.703125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5205078125 - response/aborted_ratio:0.0 - prompt_length/mean:62.0 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:4.1342367619945435 - timing_s/old_log_prob:2.3925661260000197 - timing_s/adv:0.18742299899895443 - timing_s/update_actor:4.817277075002494 - timing_s/update_weights:0.9944432230040547 - timing_s/step:12.699193484993884 - timing_per_token_ms/adv:0.0011680065248214828 - timing_per_token_ms/update_actor:0.03002092104772718 - timing_per_token_ms/gen:0.04263154555760749 - perf/total_num_tokens:160464 - perf/time_per_step:12.699193484993884 - perf/throughput:12635.763065553236 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:32 - global_seqlen/min:157446.0 - global_seqlen/max:157446.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157446.0 - global_seqlen/balanced_max:157446.0 - global_seqlen/mean:157446.0 - actor/entropy:1.6544220447540283 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.25921666622161865 - training/rollout_probs_diff_mean:0.006838637869805098 - training/rollout_probs_diff_std:0.011339455842971802 - training/rollout_actor_probs_pearson_corr:0.999442458152771 - rollout_corr/training_ppl:9.777950286865234 - rollout_corr/training_log_ppl:1.6933552026748657 - rollout_corr/kl:0.0016432899283245206 - rollout_corr/k3_kl:0.0016175456112250686 - rollout_corr/rollout_ppl:9.798437118530273 - rollout_corr/rollout_log_ppl:1.69158136844635 - rollout_corr/log_ppl_diff:0.0017738711321726441 - rollout_corr/log_ppl_abs_diff:0.005961256567388773 - rollout_corr/log_ppl_diff_max:0.09577417373657227 - rollout_corr/log_ppl_diff_min:-0.09114551544189453 - rollout_corr/ppl_ratio:1.0018181800842285 - rollout_corr/chi2_token:0.0032074451446533203 - rollout_corr/chi2_seq:0.3239586353302002 - actor/pg_clipfrac:0.0007029102107480867 - actor/ppo_kl:-7.25008053450438e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.006993902987233014 - actor/loss:-0.006993903080001473 - actor/grad_norm:0.235595703125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.2547287940979 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:32 - training/epoch:0 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.009506546892225742 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.009506546892225742 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.037109375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.484375 - response_length_non_aborted/mean:92.037109375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.484375 - response/aborted_ratio:0.0 - prompt_length/mean:61.71875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.146582436005701 - timing_s/old_log_prob:2.3419729829984135 - timing_s/adv:0.18780587500077672 - timing_s/update_actor:4.810331010005029 - timing_s/update_weights:0.9912012420027168 - timing_s/step:12.649035531001573 - timing_per_token_ms/adv:0.0011928272233068908 - timing_per_token_ms/update_actor:0.03055225925082269 - timing_per_token_ms/gen:0.04399743687801818 - perf/total_num_tokens:157446 - perf/time_per_step:12.649035531001573 - perf/throughput:12447.273123244453 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 9%|▉ | 32/350 [07:27<1:12:50, 13.74s/it] (TaskRunnerV1 pid=2350) Training Progress: 9%|▉ | 33/350 [07:40<1:11:27, 13.52s/it] (TaskRunnerV1 pid=2350) Training Progress: 10%|▉ | 34/350 [07:53<1:10:16, 13.34s/it] (TaskRunnerV1 pid=2350) step:33 - global_seqlen/min:160082.0 - global_seqlen/max:160082.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160082.0 - global_seqlen/balanced_max:160082.0 - global_seqlen/mean:160082.0 - actor/entropy:1.6527503728866577 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2123400866985321 - training/rollout_probs_diff_mean:0.006779402028769255 - training/rollout_probs_diff_std:0.01115608960390091 - training/rollout_actor_probs_pearson_corr:0.9994616508483887 - rollout_corr/training_ppl:10.107381820678711 - rollout_corr/training_log_ppl:1.7445025444030762 - rollout_corr/kl:0.0014290324179455638 - rollout_corr/k3_kl:0.0016288929618895054 - rollout_corr/rollout_ppl:10.036290168762207 - rollout_corr/rollout_log_ppl:1.7426255941390991 - rollout_corr/log_ppl_diff:0.0018770383903756738 - rollout_corr/log_ppl_abs_diff:0.00648530526086688 - rollout_corr/log_ppl_diff_max:0.08544862270355225 - rollout_corr/log_ppl_diff_min:-0.1010596752166748 - rollout_corr/ppl_ratio:1.0019338130950928 - rollout_corr/chi2_token:0.003676772117614746 - rollout_corr/chi2_seq:0.42618072032928467 - actor/pg_clipfrac:0.0008329265577534291 - actor/ppo_kl:0.00012751214944728417 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.005543768056668341 - actor/loss:0.005543768114876002 - actor/grad_norm:0.22314453125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.36434030532837 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:33 - training/epoch:0 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0012109169038012624 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.0012109169038012624 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.814453125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.5 - response_length_non_aborted/mean:92.814453125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.5 - response/aborted_ratio:0.0 - prompt_length/mean:63.515625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.130657108005835 - timing_s/old_log_prob:2.527214105000894 - timing_s/adv:0.19514899200294167 - timing_s/update_actor:4.8534739919996355 - timing_s/update_weights:0.9930089739937102 - timing_s/step:12.883812661995762 - timing_per_token_ms/adv:0.0012190564335961675 - timing_per_token_ms/update_actor:0.03031867412950635 - timing_per_token_ms/gen:0.04346138662913065 - perf/total_num_tokens:160082 - perf/time_per_step:12.883812661995762 - perf/throughput:12425.048718086728 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:34 - global_seqlen/min:162158.0 - global_seqlen/max:162158.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:162158.0 - global_seqlen/balanced_max:162158.0 - global_seqlen/mean:162158.0 - actor/entropy:1.6493852138519287 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2329278588294983 - training/rollout_probs_diff_mean:0.00681479973718524 - training/rollout_probs_diff_std:0.01137611549347639 - training/rollout_actor_probs_pearson_corr:0.9994447827339172 - rollout_corr/training_ppl:10.226446151733398 - rollout_corr/training_log_ppl:1.7157618999481201 - rollout_corr/kl:0.001763492007739842 - rollout_corr/k3_kl:0.00162686372641474 - rollout_corr/rollout_ppl:10.180974960327148 - rollout_corr/rollout_log_ppl:1.713493824005127 - rollout_corr/log_ppl_diff:0.0022681052796542645 - rollout_corr/log_ppl_abs_diff:0.006035413593053818 - rollout_corr/log_ppl_diff_max:0.09728503227233887 - rollout_corr/log_ppl_diff_min:-0.04870963096618652 - rollout_corr/ppl_ratio:1.0023162364959717 - rollout_corr/chi2_token:0.002980351448059082 - rollout_corr/chi2_seq:0.25826597213745117 - actor/pg_clipfrac:0.0005965273621768574 - actor/ppo_kl:0.00015259837425674053 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005593754443907528 - actor/loss:-0.005593754816800356 - actor/grad_norm:0.21636962890625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.447138786315918 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:34 - training/epoch:0 - critic/score/mean:0.029296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.029296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0051690395921468735 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0051690395921468735 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:95.216796875 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.5146484375 - response_length_non_aborted/mean:95.216796875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.5146484375 - response/aborted_ratio:0.0 - prompt_length/mean:63.140625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.142121299999417 - timing_s/old_log_prob:2.334459739999147 - timing_s/adv:0.38050504100101534 - timing_s/update_actor:4.755367408004531 - timing_s/update_weights:0.998521018998872 - timing_s/step:12.796611393998319 - timing_per_token_ms/adv:0.0023465079798777448 - timing_per_token_ms/update_actor:0.029325518370999466 - timing_per_token_ms/gen:0.042482423950271964 - perf/total_num_tokens:162158 - perf/time_per_step:12.796611393998319 - perf/throughput:12671.94845629625 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:35 - global_seqlen/min:159353.0 - global_seqlen/max:159353.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159353.0 - global_seqlen/balanced_max:159353.0 - global_seqlen/mean:159353.0 - actor/entropy:1.702156662940979 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2581028938293457 - training/rollout_probs_diff_mean:0.006732647772878408 - training/rollout_probs_diff_std:0.01108773984014988 - training/rollout_actor_probs_pearson_corr:0.99947190284729 - rollout_corr/training_ppl:11.641549110412598 - rollout_corr/training_log_ppl:1.7566766738891602 - rollout_corr/kl:0.0016916909953579307 - rollout_corr/k3_kl:0.001658296794630587 - rollout_corr/rollout_ppl:11.621134757995605 - rollout_corr/rollout_log_ppl:1.7547240257263184 - rollout_corr/log_ppl_diff:0.0019525944953784347 - rollout_corr/log_ppl_abs_diff:0.006335945334285498 - rollout_corr/log_ppl_diff_max:0.06381797790527344 - rollout_corr/log_ppl_diff_min:-0.07049918174743652 - rollout_corr/ppl_ratio:1.0019967555999756 - rollout_corr/chi2_token:0.0032809972763061523 - rollout_corr/chi2_seq:0.4474446773529053 - actor/pg_clipfrac:0.0007547000465516854 - actor/ppo_kl:2.8150648176961113e-06 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.009549780545057729 - actor/loss:-0.00954978121444583 - actor/grad_norm:0.2391357421875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.5270414352417 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.031) - val-core/char_count/acc/mean@1:np.float64(0.031) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:35 - training/epoch:0 - critic/score/mean:0.02734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.02734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.002526399912312627 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.002526399912312627 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.8056640625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5048828125 - response_length_non_aborted/mean:93.8056640625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5048828125 - response/aborted_ratio:0.0 - prompt_length/mean:61.8125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.141578201997618 - timing_s/old_log_prob:2.3254059219980263 - timing_s/adv:0.37963998100167373 - timing_s/update_actor:4.818912508999347 - timing_s/update_weights:0.998465133998252 - timing_s/step:12.843311892000202 - timing_s/testing:4.637923674003105 - timing_per_token_ms/adv:0.0023823836451254366 - timing_per_token_ms/update_actor:0.030240488155223604 - timing_per_token_ms/gen:0.043115839574394556 - perf/total_num_tokens:159353 - perf/time_per_step:12.843311892000202 - perf/throughput:12407.469454919743 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 10%|█ | 35/350 [08:11<1:16:46, 14.62s/it] (TaskRunnerV1 pid=2350) Training Progress: 10%|█ | 36/350 [08:24<1:14:00, 14.14s/it] (TaskRunnerV1 pid=2350) Training Progress: 11%|█ | 37/350 [08:37<1:12:01, 13.81s/it] (TaskRunnerV1 pid=2350) step:36 - global_seqlen/min:163825.0 - global_seqlen/max:163825.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:163825.0 - global_seqlen/balanced_max:163825.0 - global_seqlen/mean:163825.0 - actor/entropy:1.6951159238815308 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.1795732080936432 - training/rollout_probs_diff_mean:0.006663641892373562 - training/rollout_probs_diff_std:0.011221051216125488 - training/rollout_actor_probs_pearson_corr:0.9994692206382751 - rollout_corr/training_ppl:10.601774215698242 - rollout_corr/training_log_ppl:1.7698462009429932 - rollout_corr/kl:0.001672546030022204 - rollout_corr/k3_kl:0.0016585005214437842 - rollout_corr/rollout_ppl:10.56168270111084 - rollout_corr/rollout_log_ppl:1.7679128646850586 - rollout_corr/log_ppl_diff:0.0019332116935402155 - rollout_corr/log_ppl_abs_diff:0.00604025786742568 - rollout_corr/log_ppl_diff_max:0.12730860710144043 - rollout_corr/log_ppl_diff_min:-0.040993690490722656 - rollout_corr/ppl_ratio:1.0019783973693848 - rollout_corr/chi2_token:0.0033038854598999023 - rollout_corr/chi2_seq:0.3570590019226074 - actor/pg_clipfrac:0.0006776836101683632 - actor/ppo_kl:1.0421606930322014e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0020914978813380003 - actor/loss:-0.002091497997753322 - actor/grad_norm:0.2508544921875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.40650510787964 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:36 - training/epoch:0 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.003448149422183633 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.003448149422183633 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:95.9697265625 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.5322265625 - response_length_non_aborted/mean:95.9697265625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.5322265625 - response/aborted_ratio:0.0 - prompt_length/mean:64.015625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:4.122847147998982 - timing_s/old_log_prob:2.3470285389994388 - timing_s/adv:0.18938849899859633 - timing_s/update_actor:4.852330984002037 - timing_s/update_weights:0.9931782119965646 - timing_s/step:12.892751446997863 - timing_per_token_ms/adv:0.0011560415015937514 - timing_per_token_ms/update_actor:0.029618989678022508 - timing_per_token_ms/gen:0.04195299978629921 - perf/total_num_tokens:163825 - perf/time_per_step:12.892751446997863 - perf/throughput:12706.752369615207 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:37 - global_seqlen/min:159327.0 - global_seqlen/max:159327.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159327.0 - global_seqlen/balanced_max:159327.0 - global_seqlen/mean:159327.0 - actor/entropy:1.6719733476638794 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2514486014842987 - training/rollout_probs_diff_mean:0.006890688091516495 - training/rollout_probs_diff_std:0.01140675600618124 - training/rollout_actor_probs_pearson_corr:0.9994346499443054 - rollout_corr/training_ppl:9.970866203308105 - rollout_corr/training_log_ppl:1.7258572578430176 - rollout_corr/kl:0.002098593395203352 - rollout_corr/k3_kl:0.00167076475918293 - rollout_corr/rollout_ppl:9.931450843811035 - rollout_corr/rollout_log_ppl:1.7238503694534302 - rollout_corr/log_ppl_diff:0.002006965223699808 - rollout_corr/log_ppl_abs_diff:0.0064652590081095695 - rollout_corr/log_ppl_diff_max:0.06246328353881836 - rollout_corr/log_ppl_diff_min:-0.08687210083007812 - rollout_corr/ppl_ratio:1.0020537376403809 - rollout_corr/chi2_token:0.0025191307067871094 - rollout_corr/chi2_seq:0.29522812366485596 - actor/pg_clipfrac:0.0004808120308249878 - actor/ppo_kl:-0.0005479919609570061 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.006536466564284638 - actor/loss:-0.0065364663023501635 - actor/grad_norm:0.21502685546875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.411307334899902 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:37 - training/epoch:0 - critic/score/mean:0.025390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.025390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.011411786079406738 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.011411786079406738 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.2333984375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4970703125 - response_length_non_aborted/mean:93.2333984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4970703125 - response/aborted_ratio:0.0 - prompt_length/mean:62.359375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.144947228996898 - timing_s/old_log_prob:2.5833986979996553 - timing_s/adv:0.18235300300148083 - timing_s/update_actor:4.826191809996089 - timing_s/update_weights:0.9909486129981815 - timing_s/step:12.902561394999793 - timing_per_token_ms/adv:0.0011445204077242452 - timing_per_token_ms/update_actor:0.03029111079726656 - timing_per_token_ms/gen:0.043415772632494666 - perf/total_num_tokens:159327 - perf/time_per_step:12.902561394999793 - perf/throughput:12348.47834645801 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:38 - global_seqlen/min:160105.0 - global_seqlen/max:160105.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160105.0 - global_seqlen/balanced_max:160105.0 - global_seqlen/mean:160105.0 - actor/entropy:1.7358622550964355 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3974529206752777 - training/rollout_probs_diff_mean:0.006799974013119936 - training/rollout_probs_diff_std:0.01126931793987751 - training/rollout_actor_probs_pearson_corr:0.9994523525238037 - rollout_corr/training_ppl:11.22261905670166 - rollout_corr/training_log_ppl:1.8063154220581055 - rollout_corr/kl:0.0018000536365434527 - rollout_corr/k3_kl:0.0016914535081014037 - rollout_corr/rollout_ppl:11.21077823638916 - rollout_corr/rollout_log_ppl:1.8042981624603271 - rollout_corr/log_ppl_diff:0.0020173201337456703 - rollout_corr/log_ppl_abs_diff:0.006087939720600843 - rollout_corr/log_ppl_diff_max:0.05081677436828613 - rollout_corr/log_ppl_diff_min:-0.05363178253173828 - rollout_corr/ppl_ratio:1.002058982849121 - rollout_corr/chi2_token:0.0031965970993041992 - rollout_corr/chi2_seq:0.2723879814147949 - actor/pg_clipfrac:0.0007716382618430847 - actor/ppo_kl:-0.00021679624607410614 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.01369263231572404 - actor/loss:-0.01369263231754303 - actor/grad_norm:0.2703857421875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.469759464263916 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:38 - training/epoch:0 - critic/score/mean:0.0361328125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0361328125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.018207425251603127 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.018207425251603127 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.2587890625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.51953125 - response_length_non_aborted/mean:94.2587890625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.51953125 - response/aborted_ratio:0.0 - prompt_length/mean:62.09375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:4.1338465600056225 - timing_s/old_log_prob:2.513779542998236 - timing_s/adv:0.19699355299962917 - timing_s/update_actor:4.881960245002119 - timing_s/update_weights:1.0197335189950536 - timing_s/step:12.934603729001537 - timing_per_token_ms/adv:0.0012304022547679908 - timing_per_token_ms/update_actor:0.03049224099810824 - timing_per_token_ms/gen:0.042828468001840245 - perf/total_num_tokens:160105 - perf/time_per_step:12.934603729001537 - perf/throughput:12378.036726476428 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 11%|█ | 38/350 [08:50<1:10:43, 13.60s/it] (TransferQueueController pid=2570) 2026-11-01 16:59:09,711 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_ebda4368: [Performance] Total success requests: 107398, Total req/min: 21148.45, Total avg process time: 0.0001s; (TransferQueueController pid=2570) Time range: last 5.08 minutes; (TransferQueueController pid=2570) Per-operation statistics: KV_RETRIEVE_META: req_count=40400, req/min=7955.43, avg_time=0.000259s, max_time=0.067209s, min_time=0.000062s; CLEAR_META: req_count=52, req/min=10.24, avg_time=0.001707s, max_time=0.003646s, min_time=0.000749s; SET_CUSTOM_META: req_count=40252, req/min=7926.29, avg_time=0.000010s, max_time=0.000328s, min_time=0.000003s; KV_LIST: req_count=78, req/min=15.36, avg_time=0.000316s, max_time=0.001158s, min_time=0.000111s; NOTIFY_DATA_UPDATE: req_count=26594, req/min=5236.80, avg_time=0.000088s, max_time=0.001481s, min_time=0.000030s; KV_RETRIEVE_KEYS: req_count=22, req/min=4.33, avg_time=0.000135s, max_time=0.000239s, min_time=0.000103s (SimpleStorageUnit pid=2676) Per-operation statistics: CLEAR_DATA: req_count=52, req/min=10.24, avg_time=0.001325s, max_time=0.004123s, min_time=0.000396s; PUT_DATA: req_count=3373, req/min=664.20, avg_time=0.000595s, max_time=0.001614s, min_time=0.000310s; GET_DATA: req_count=118, req/min=23.24, avg_time=0.000601s, max_time=0.001232s, min_time=0.000370s (TaskRunnerV1 pid=2350) Training Progress: 11%|█ | 39/350 [09:03<1:09:37, 13.43s/it] (TaskRunnerV1 pid=2350) Training Progress: 11%|█▏ | 40/350 [09:21<1:16:11, 14.75s/it] (SimpleStorageUnit pid=2691) 2026-11-01 16:59:09,721 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_52739293: [Performance] Total success requests: 3551, Total req/min: 699.26, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2691) Time range: last 5.08 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2691) Per-operation statistics: CLEAR_DATA: req_count=52, req/min=10.24, avg_time=0.001504s, max_time=0.004363s, min_time=0.000379s; PUT_DATA: req_count=3381, req/min=665.78, avg_time=0.000592s, max_time=0.002328s, min_time=0.000316s; GET_DATA: req_count=118, req/min=23.24, avg_time=0.000617s, max_time=0.001260s, min_time=0.000380s [repeated 7x across cluster] (TaskRunnerV1 pid=2350) step:39 - global_seqlen/min:159686.0 - global_seqlen/max:159686.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159686.0 - global_seqlen/balanced_max:159686.0 - global_seqlen/mean:159686.0 - actor/entropy:1.635151982307434 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.24198371171951294 - training/rollout_probs_diff_mean:0.006834500469267368 - training/rollout_probs_diff_std:0.011369983665645123 - training/rollout_actor_probs_pearson_corr:0.9994471669197083 - rollout_corr/training_ppl:9.57181167602539 - rollout_corr/training_log_ppl:1.6947271823883057 - rollout_corr/kl:0.0015714826295152307 - rollout_corr/k3_kl:0.0016789732035249472 - rollout_corr/rollout_ppl:9.564315795898438 - rollout_corr/rollout_log_ppl:1.6931312084197998 - rollout_corr/log_ppl_diff:0.0015961064491420984 - rollout_corr/log_ppl_abs_diff:0.006291280500590801 - rollout_corr/log_ppl_diff_max:0.054418325424194336 - rollout_corr/log_ppl_diff_min:-0.05714225769042969 - rollout_corr/ppl_ratio:1.0016402006149292 - rollout_corr/chi2_token:0.0036002397537231445 - rollout_corr/chi2_seq:0.45795226097106934 - actor/pg_clipfrac:0.0009631537844446333 - actor/ppo_kl:0.0001600628970569081 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.004611461023159791 - actor/loss:-0.004611461539752781 - actor/grad_norm:0.2354736328125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.455300331115723 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:39 - training/epoch:0 - critic/score/mean:0.0263671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0263671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00044982603867538273 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.00044982603867538273 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.708984375 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4912109375 - response_length_non_aborted/mean:92.708984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4912109375 - response/aborted_ratio:0.0 - prompt_length/mean:63.234375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:4.138228732001153 - timing_s/old_log_prob:2.3790825749965734 - timing_s/adv:0.38285852399712894 - timing_s/update_actor:4.828391102004389 - timing_s/update_weights:0.9937708390061744 - timing_s/step:12.901967915997375 - timing_per_token_ms/adv:0.0023975710080854236 - timing_per_token_ms/update_actor:0.030236784076277123 - timing_per_token_ms/gen:0.04359058642847823 - perf/total_num_tokens:159686 - perf/time_per_step:12.901967915997375 - perf/throughput:12376.871578017377 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:40 - global_seqlen/min:159829.0 - global_seqlen/max:159829.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159829.0 - global_seqlen/balanced_max:159829.0 - global_seqlen/mean:159829.0 - actor/entropy:1.701931118965149 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2811149060726166 - training/rollout_probs_diff_mean:0.0067613148130476475 - training/rollout_probs_diff_std:0.011088074184954166 - training/rollout_actor_probs_pearson_corr:0.9994659423828125 - rollout_corr/training_ppl:9.507850646972656 - rollout_corr/training_log_ppl:1.7279503345489502 - rollout_corr/kl:0.001557977288030088 - rollout_corr/k3_kl:0.00163780408911407 - rollout_corr/rollout_ppl:9.479681968688965 - rollout_corr/rollout_log_ppl:1.7265148162841797 - rollout_corr/log_ppl_diff:0.0014356207102537155 - rollout_corr/log_ppl_abs_diff:0.005751871038228273 - rollout_corr/log_ppl_diff_max:0.1017293930053711 - rollout_corr/log_ppl_diff_min:-0.0666043758392334 - rollout_corr/ppl_ratio:1.0014731884002686 - rollout_corr/chi2_token:0.0034437179565429688 - rollout_corr/chi2_seq:0.3801078796386719 - actor/pg_clipfrac:0.0006501678490167251 - actor/ppo_kl:0.00031300891532737296 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005569294441102102 - actor/loss:-0.005569294793531299 - actor/grad_norm:0.22308349609375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.440993785858154 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.027) - val-core/char_count/acc/mean@1:np.float64(0.027) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:40 - training/epoch:0 - critic/score/mean:0.02734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.02734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0014577393885701895 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0014577393885701895 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.7080078125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4873046875 - response_length_non_aborted/mean:93.7080078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4873046875 - response/aborted_ratio:0.0 - prompt_length/mean:62.375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.13452267300454 - timing_s/old_log_prob:2.4361306600039825 - timing_s/adv:0.3785507550055627 - timing_s/update_actor:4.880462686000101 - timing_s/update_weights:1.0026784460060298 - timing_s/step:13.005489027003932 - timing_s/testing:4.685216416997719 - timing_per_token_ms/adv:0.0023684735248644655 - timing_per_token_ms/update_actor:0.03053552663158814 - timing_per_token_ms/gen:0.04308724400517461 - perf/total_num_tokens:159829 - perf/time_per_step:13.005489027003932 - perf/throughput:12289.349494520293 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:41 - global_seqlen/min:158438.0 - global_seqlen/max:158438.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158438.0 - global_seqlen/balanced_max:158438.0 - global_seqlen/mean:158438.0 - actor/entropy:1.7039209604263306 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.23903638124465942 - training/rollout_probs_diff_mean:0.0067861308343708515 - training/rollout_probs_diff_std:0.011185669340193272 - training/rollout_actor_probs_pearson_corr:0.9994606375694275 - rollout_corr/training_ppl:10.15626049041748 - rollout_corr/training_log_ppl:1.7477813959121704 - rollout_corr/kl:0.002016175538301468 - rollout_corr/k3_kl:0.0016481721540912986 - rollout_corr/rollout_ppl:10.124650001525879 - rollout_corr/rollout_log_ppl:1.7458581924438477 - rollout_corr/log_ppl_diff:0.0019231283804401755 - rollout_corr/log_ppl_abs_diff:0.006342739798128605 - rollout_corr/log_ppl_diff_max:0.08141136169433594 - rollout_corr/log_ppl_diff_min:-0.10042333602905273 - rollout_corr/ppl_ratio:1.0019742250442505 - rollout_corr/chi2_token:0.002553105354309082 - rollout_corr/chi2_seq:0.3346458673477173 - actor/pg_clipfrac:0.0006090308161219582 - actor/ppo_kl:-0.00027520585210064985 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.00010219436080660671 - actor/loss:-0.00010219460818916559 - actor/grad_norm:0.19305419921875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.292893409729004 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:41 - training/epoch:0 - critic/score/mean:0.0205078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0205078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0015982460463419557 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.0015982460463419557 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.990234375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.48828125 - response_length_non_aborted/mean:92.990234375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.48828125 - response/aborted_ratio:0.0 - prompt_length/mean:61.734375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.142554056001245 - timing_s/old_log_prob:2.5395309230007115 - timing_s/adv:0.18453525899531087 - timing_s/update_actor:4.823534145005397 - timing_s/update_weights:0.9951086090004537 - timing_s/step:12.861224432999734 - timing_per_token_ms/adv:0.0011647159077703005 - timing_per_token_ms/update_actor:0.03044430089375905 - timing_per_token_ms/gen:0.04350416979270804 - perf/total_num_tokens:158438 - perf/time_per_step:12.861224432999734 - perf/throughput:12319.04480210102 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 12%|█▏ | 41/350 [09:34<1:13:19, 14.24s/it] (TaskRunnerV1 pid=2350) Training Progress: 12%|█▏ | 42/350 [09:47<1:11:15, 13.88s/it] (TaskRunnerV1 pid=2350) Training Progress: 12%|█▏ | 43/350 [10:00<1:09:41, 13.62s/it] (TaskRunnerV1 pid=2350) step:42 - global_seqlen/min:158976.0 - global_seqlen/max:158976.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158976.0 - global_seqlen/balanced_max:158976.0 - global_seqlen/mean:158976.0 - actor/entropy:1.6467649936676025 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.354892373085022 - training/rollout_probs_diff_mean:0.006561031565070152 - training/rollout_probs_diff_std:0.010894537903368473 - training/rollout_actor_probs_pearson_corr:0.9994938969612122 - rollout_corr/training_ppl:9.425863265991211 - rollout_corr/training_log_ppl:1.7190849781036377 - rollout_corr/kl:0.001846575178205967 - rollout_corr/k3_kl:0.0016072381986305118 - rollout_corr/rollout_ppl:9.420294761657715 - rollout_corr/rollout_log_ppl:1.717149257659912 - rollout_corr/log_ppl_diff:0.001935895998030901 - rollout_corr/log_ppl_abs_diff:0.005835093092173338 - rollout_corr/log_ppl_diff_max:0.058176398277282715 - rollout_corr/log_ppl_diff_min:-0.08831501007080078 - rollout_corr/ppl_ratio:1.001975655555725 - rollout_corr/chi2_token:0.002729177474975586 - rollout_corr/chi2_seq:0.23947083950042725 - actor/pg_clipfrac:0.0005739472496012846 - actor/ppo_kl:0.00025756488381982 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.003298238198112813 - actor/loss:-0.0032982382690533996 - actor/grad_norm:0.218017578125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.32973623275757 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:42 - training/epoch:0 - critic/score/mean:0.025390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.025390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.004519272595643997 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.004519272595643997 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:92.9375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4970703125 - response_length_non_aborted/mean:92.9375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4970703125 - response/aborted_ratio:0.0 - prompt_length/mean:62.3125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.1357277810020605 - timing_s/old_log_prob:2.317848443002731 - timing_s/adv:0.37871990500570973 - timing_s/update_actor:4.8753358100002515 - timing_s/update_weights:1.012648898999032 - timing_s/step:12.932406220999837 - timing_per_token_ms/adv:0.002382245779273033 - timing_per_token_ms/update_actor:0.03066711837007002 - timing_per_token_ms/gen:0.04345712614536462 - perf/total_num_tokens:158976 - perf/time_per_step:12.932406220999837 - perf/throughput:12292.839962129581 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:43 - global_seqlen/min:159256.0 - global_seqlen/max:159256.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159256.0 - global_seqlen/balanced_max:159256.0 - global_seqlen/mean:159256.0 - actor/entropy:1.7585039138793945 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.26965731382369995 - training/rollout_probs_diff_mean:0.0066752382554113865 - training/rollout_probs_diff_std:0.011039265431463718 - training/rollout_actor_probs_pearson_corr:0.9994795918464661 - rollout_corr/training_ppl:11.492973327636719 - rollout_corr/training_log_ppl:1.8256875276565552 - rollout_corr/kl:0.0013501144712790847 - rollout_corr/k3_kl:0.0016763235908001661 - rollout_corr/rollout_ppl:11.496757507324219 - rollout_corr/rollout_log_ppl:1.8244669437408447 - rollout_corr/log_ppl_diff:0.0012204879894852638 - rollout_corr/log_ppl_abs_diff:0.00596452783793211 - rollout_corr/log_ppl_diff_max:0.03656172752380371 - rollout_corr/log_ppl_diff_min:-0.046910762786865234 - rollout_corr/ppl_ratio:1.0012550354003906 - rollout_corr/chi2_token:0.004023909568786621 - rollout_corr/chi2_seq:0.42757701873779297 - actor/pg_clipfrac:0.0007635797689241979 - actor/ppo_kl:2.8609498258447285e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.004443665035068989 - actor/loss:0.004443665035068989 - actor/grad_norm:0.2227783203125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.371252059936523 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:43 - training/epoch:0 - critic/score/mean:0.0263671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0263671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004403398837894201 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.004403398837894201 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.8671875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.501953125 - response_length_non_aborted/mean:92.8671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.501953125 - response/aborted_ratio:0.0 - prompt_length/mean:62.65625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.147018869996828 - timing_s/old_log_prob:2.3447090970003046 - timing_s/adv:0.38591726899903733 - timing_s/update_actor:4.827307915998972 - timing_s/update_weights:0.999386842995591 - timing_s/step:12.883746018000238 - timing_per_token_ms/adv:0.0024232510486200667 - timing_per_token_ms/update_actor:0.03031162352438195 - timing_per_token_ms/gen:0.04360876240848014 - perf/total_num_tokens:159256 - perf/time_per_step:12.883746018000238 - perf/throughput:12361.001200854087 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:44 - global_seqlen/min:158526.0 - global_seqlen/max:158526.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158526.0 - global_seqlen/balanced_max:158526.0 - global_seqlen/mean:158526.0 - actor/entropy:1.6799601316452026 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.17507919669151306 - training/rollout_probs_diff_mean:0.006737450137734413 - training/rollout_probs_diff_std:0.011070559732615948 - training/rollout_actor_probs_pearson_corr:0.9994704127311707 - rollout_corr/training_ppl:10.749258041381836 - rollout_corr/training_log_ppl:1.7519484758377075 - rollout_corr/kl:0.0016429551178589463 - rollout_corr/k3_kl:0.0016506972024217248 - rollout_corr/rollout_ppl:10.723069190979004 - rollout_corr/rollout_log_ppl:1.7499502897262573 - rollout_corr/log_ppl_diff:0.001998203108087182 - rollout_corr/log_ppl_abs_diff:0.006094654556363821 - rollout_corr/log_ppl_diff_max:0.11403465270996094 - rollout_corr/log_ppl_diff_min:-0.1261272430419922 - rollout_corr/ppl_ratio:1.0020487308502197 - rollout_corr/chi2_token:0.0033293962478637695 - rollout_corr/chi2_seq:0.27655506134033203 - actor/pg_clipfrac:0.0005879722478615197 - actor/ppo_kl:9.973724506077512e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.004174911075097043 - actor/loss:-0.004174911184236407 - actor/grad_norm:0.2237548828125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.40520668029785 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:44 - training/epoch:0 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.007011216599494219 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.007011216599494219 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.388671875 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.478515625 - response_length_non_aborted/mean:92.388671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.478515625 - response/aborted_ratio:0.0 - prompt_length/mean:62.421875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.124035171000287 - timing_s/old_log_prob:2.338245936000021 - timing_s/adv:0.18152145399653818 - timing_s/update_actor:5.021003400994232 - timing_s/update_weights:1.0040410990040982 - timing_s/step:12.847049956995761 - timing_per_token_ms/adv:0.0011450579336925058 - timing_per_token_ms/update_actor:0.031673059315154815 - timing_per_token_ms/gen:0.04359168732427422 - perf/total_num_tokens:158526 - perf/time_per_step:12.847049956995761 - perf/throughput:12339.486538205287 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 13%|█▎ | 44/350 [10:13<1:08:28, 13.43s/it] (TaskRunnerV1 pid=2350) Training Progress: 13%|█▎ | 45/350 [10:31<1:14:40, 14.69s/it] (TaskRunnerV1 pid=2350) Training Progress: 13%|█▎ | 46/350 [10:44<1:12:00, 14.21s/it] (TaskRunnerV1 pid=2350) step:45 - global_seqlen/min:159724.0 - global_seqlen/max:159724.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159724.0 - global_seqlen/balanced_max:159724.0 - global_seqlen/mean:159724.0 - actor/entropy:1.697266697883606 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2863205671310425 - training/rollout_probs_diff_mean:0.006781135685741901 - training/rollout_probs_diff_std:0.011151342652738094 - training/rollout_actor_probs_pearson_corr:0.9994606375694275 - rollout_corr/training_ppl:10.339035034179688 - rollout_corr/training_log_ppl:1.7728570699691772 - rollout_corr/kl:0.0016741125145927072 - rollout_corr/k3_kl:0.0016460706247016788 - rollout_corr/rollout_ppl:10.312337875366211 - rollout_corr/rollout_log_ppl:1.77079439163208 - rollout_corr/log_ppl_diff:0.00206259498372674 - rollout_corr/log_ppl_abs_diff:0.00640895077958703 - rollout_corr/log_ppl_diff_max:0.11653614044189453 - rollout_corr/log_ppl_diff_min:-0.08528447151184082 - rollout_corr/ppl_ratio:1.002117395401001 - rollout_corr/chi2_token:0.0032434463500976562 - rollout_corr/chi2_seq:0.364182710647583 - actor/pg_clipfrac:0.000571498505451018 - actor/ppo_kl:-0.00022913485820671064 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.003547617990989238 - actor/loss:-0.0035476182238198817 - actor/grad_norm:0.19879150390625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:30.501348972320557 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.031) - val-core/char_count/acc/mean@1:np.float64(0.031) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:45 - training/epoch:0 - critic/score/mean:0.0205078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0205078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.009385906159877777 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.009385906159877777 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.41796875 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.5244140625 - response_length_non_aborted/mean:94.41796875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.5244140625 - response/aborted_ratio:0.0 - prompt_length/mean:61.5625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:4.147590524000407 - timing_s/old_log_prob:2.3372894220010494 - timing_s/adv:0.18286988000181736 - timing_s/update_actor:4.825660544993298 - timing_s/update_weights:1.0116709420035477 - timing_s/step:12.675424959998054 - timing_s/testing:4.838418910003384 - timing_per_token_ms/adv:0.001144911722733073 - timing_per_token_ms/update_actor:0.030212494960014134 - timing_per_token_ms/gen:0.04289841673907169 - perf/total_num_tokens:159724 - perf/time_per_step:12.675424959998054 - perf/throughput:12601.076532271509 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:46 - global_seqlen/min:160730.0 - global_seqlen/max:160730.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160730.0 - global_seqlen/balanced_max:160730.0 - global_seqlen/mean:160730.0 - actor/entropy:1.6366568803787231 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.22857919335365295 - training/rollout_probs_diff_mean:0.006804768927395344 - training/rollout_probs_diff_std:0.011022834107279778 - training/rollout_actor_probs_pearson_corr:0.9994661808013916 - rollout_corr/training_ppl:10.617775917053223 - rollout_corr/training_log_ppl:1.7186594009399414 - rollout_corr/kl:0.0016499231569468975 - rollout_corr/k3_kl:0.0016303188167512417 - rollout_corr/rollout_ppl:10.641780853271484 - rollout_corr/rollout_log_ppl:1.717310905456543 - rollout_corr/log_ppl_diff:0.0013486305251717567 - rollout_corr/log_ppl_abs_diff:0.00616292841732502 - rollout_corr/log_ppl_diff_max:0.06109046936035156 - rollout_corr/log_ppl_diff_min:-0.13474321365356445 - rollout_corr/ppl_ratio:1.0014011859893799 - rollout_corr/chi2_token:0.0032540559768676758 - rollout_corr/chi2_seq:0.35126733779907227 - actor/pg_clipfrac:0.00044629029313606833 - actor/ppo_kl:8.257640179939094e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0030662439385196194 - actor/loss:0.0030662432545796037 - actor/grad_norm:0.1669921875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.285162448883057 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:46 - training/epoch:0 - critic/score/mean:0.021484375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.021484375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.001300451927818358 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.001300451927818358 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:95.509765625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5361328125 - response_length_non_aborted/mean:95.509765625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5361328125 - response/aborted_ratio:0.0 - prompt_length/mean:61.453125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.135269655998854 - timing_s/old_log_prob:2.3619729670026572 - timing_s/adv:0.37363906200334895 - timing_s/update_actor:4.8617967999962275 - timing_s/update_weights:0.997331447993929 - timing_s/step:12.907090016000438 - timing_per_token_ms/adv:0.002324637976752 - timing_per_token_ms/update_actor:0.030248222484889117 - timing_per_token_ms/gen:0.04228205615425916 - perf/total_num_tokens:160730 - perf/time_per_step:12.907090016000438 - perf/throughput:12452.845668601443 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:47 - global_seqlen/min:156524.0 - global_seqlen/max:156524.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156524.0 - global_seqlen/balanced_max:156524.0 - global_seqlen/mean:156524.0 - actor/entropy:1.656773328781128 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.31865766644477844 - training/rollout_probs_diff_mean:0.006770504172891378 - training/rollout_probs_diff_std:0.011304665356874466 - training/rollout_actor_probs_pearson_corr:0.9994533658027649 - rollout_corr/training_ppl:9.330270767211914 - rollout_corr/training_log_ppl:1.712904930114746 - rollout_corr/kl:0.0017498075030744076 - rollout_corr/k3_kl:0.0016485077794641256 - rollout_corr/rollout_ppl:9.316390037536621 - rollout_corr/rollout_log_ppl:1.7111148834228516 - rollout_corr/log_ppl_diff:0.0017899785889312625 - rollout_corr/log_ppl_abs_diff:0.005775635130703449 - rollout_corr/log_ppl_diff_max:0.06952905654907227 - rollout_corr/log_ppl_diff_min:-0.03589940071105957 - rollout_corr/ppl_ratio:1.0018236637115479 - rollout_corr/chi2_token:0.003123044967651367 - rollout_corr/chi2_seq:0.24187934398651123 - actor/pg_clipfrac:0.0006115466258052038 - actor/ppo_kl:-0.00011486618313938382 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0007708696939516813 - actor/loss:0.0007708697812631726 - actor/grad_norm:0.22216796875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.376341819763184 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:47 - training/epoch:0 - critic/score/mean:0.02734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.02734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0003169141127727926 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.0003169141127727926 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.32421875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.4677734375 - response_length_non_aborted/mean:91.32421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.4677734375 - response/aborted_ratio:0.0 - prompt_length/mean:61.53125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.1015625 - timing_s/gen:4.128723727997567 - timing_s/old_log_prob:2.3468426239996916 - timing_s/adv:0.4403134849999333 - timing_s/update_actor:4.8725941110024 - timing_s/update_weights:1.0032088660009322 - timing_s/step:12.967605149002338 - timing_per_token_ms/adv:0.0028130732986630376 - timing_per_token_ms/update_actor:0.031130012720109373 - timing_per_token_ms/gen:0.04414991796053688 - perf/total_num_tokens:156524 - perf/time_per_step:12.967605149002338 - perf/throughput:12070.386027449498 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 13%|█▎ | 47/350 [10:57<1:10:04, 13.88s/it] (TaskRunnerV1 pid=2350) Training Progress: 14%|█▎ | 48/350 [11:10<1:08:43, 13.65s/it] (TaskRunnerV1 pid=2350) Training Progress: 14%|█▍ | 49/350 [11:23<1:07:40, 13.49s/it] (TaskRunnerV1 pid=2350) step:48 - global_seqlen/min:158892.0 - global_seqlen/max:158892.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158892.0 - global_seqlen/balanced_max:158892.0 - global_seqlen/mean:158892.0 - actor/entropy:1.7017329931259155 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.22998332977294922 - training/rollout_probs_diff_mean:0.00695078307762742 - training/rollout_probs_diff_std:0.011478646658360958 - training/rollout_actor_probs_pearson_corr:0.9994266033172607 - rollout_corr/training_ppl:8.982358932495117 - rollout_corr/training_log_ppl:1.7336273193359375 - rollout_corr/kl:0.001659909263253212 - rollout_corr/k3_kl:0.0017126667080447078 - rollout_corr/rollout_ppl:8.963829040527344 - rollout_corr/rollout_log_ppl:1.7318801879882812 - rollout_corr/log_ppl_diff:0.001746898633427918 - rollout_corr/log_ppl_abs_diff:0.006140268873423338 - rollout_corr/log_ppl_diff_max:0.06929731369018555 - rollout_corr/log_ppl_diff_min:-0.05358767509460449 - rollout_corr/ppl_ratio:1.001785159111023 - rollout_corr/chi2_token:0.003523707389831543 - rollout_corr/chi2_seq:0.34572577476501465 - actor/pg_clipfrac:0.0009786442484861861 - actor/ppo_kl:0.00010655850140513699 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.009757231433468405 - actor/loss:-0.00975723029114306 - actor/grad_norm:0.25 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.398784637451172 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:48 - training/epoch:0 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.011728483252227306 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.011728483252227306 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.46484375 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.5087890625 - response_length_non_aborted/mean:94.46484375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.5087890625 - response/aborted_ratio:0.0 - prompt_length/mean:60.703125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.157645292994857 - timing_s/old_log_prob:2.398353420001513 - timing_s/adv:0.18843362500047078 - timing_s/update_actor:5.052359988003445 - timing_s/update_weights:1.02267499500158 - timing_s/step:12.998384456994245 - timing_per_token_ms/adv:0.0011859226707478714 - timing_per_token_ms/update_actor:0.031797447247208455 - timing_per_token_ms/gen:0.04298107444273722 - perf/total_num_tokens:158892 - perf/time_per_step:12.998384456994245 - perf/throughput:12223.980643571631 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:49 - global_seqlen/min:160299.0 - global_seqlen/max:160299.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160299.0 - global_seqlen/balanced_max:160299.0 - global_seqlen/mean:160299.0 - actor/entropy:1.7100787162780762 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.338111937046051 - training/rollout_probs_diff_mean:0.006896674167364836 - training/rollout_probs_diff_std:0.011521493084728718 - training/rollout_actor_probs_pearson_corr:0.9994304180145264 - rollout_corr/training_ppl:11.155510902404785 - rollout_corr/training_log_ppl:1.7695531845092773 - rollout_corr/kl:0.0013748133787885308 - rollout_corr/k3_kl:0.0017251520184800029 - rollout_corr/rollout_ppl:11.125946044921875 - rollout_corr/rollout_log_ppl:1.7675100564956665 - rollout_corr/log_ppl_diff:0.002043254906311631 - rollout_corr/log_ppl_abs_diff:0.006555527914315462 - rollout_corr/log_ppl_diff_max:0.08590412139892578 - rollout_corr/log_ppl_diff_min:-0.06199455261230469 - rollout_corr/ppl_ratio:1.0020952224731445 - rollout_corr/chi2_token:0.004160881042480469 - rollout_corr/chi2_seq:0.617470383644104 - actor/pg_clipfrac:0.0008600053927996971 - actor/ppo_kl:0.00020302143623969943 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.006011957390001044 - actor/loss:-0.006011957302689552 - actor/grad_norm:0.2772216796875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.40794086456299 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:49 - training/epoch:0 - critic/score/mean:0.037109375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.037109375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.012207896448671818 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.012207896448671818 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.4169921875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5166015625 - response_length_non_aborted/mean:93.4169921875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5166015625 - response/aborted_ratio:0.0 - prompt_length/mean:63.125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.13652210500004 - timing_s/old_log_prob:2.4013167079974664 - timing_s/adv:0.18312542199419113 - timing_s/update_actor:4.8980378510023 - timing_s/update_weights:0.9987829120000242 - timing_s/step:12.801075416995445 - timing_per_token_ms/adv:0.0011423990292777318 - timing_per_token_ms/update_actor:0.030555635724504208 - timing_per_token_ms/gen:0.04324237243751283 - perf/total_num_tokens:160299 - perf/time_per_step:12.801075416995445 - perf/throughput:12522.307288899947 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:50 - global_seqlen/min:159321.0 - global_seqlen/max:159321.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159321.0 - global_seqlen/balanced_max:159321.0 - global_seqlen/mean:159321.0 - actor/entropy:1.6547214984893799 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3366926312446594 - training/rollout_probs_diff_mean:0.006690909154713154 - training/rollout_probs_diff_std:0.010996146127581596 - training/rollout_actor_probs_pearson_corr:0.999477207660675 - rollout_corr/training_ppl:10.04649543762207 - rollout_corr/training_log_ppl:1.6901952028274536 - rollout_corr/kl:0.0016385322669520974 - rollout_corr/k3_kl:0.0015949548687785864 - rollout_corr/rollout_ppl:9.979022979736328 - rollout_corr/rollout_log_ppl:1.6884791851043701 - rollout_corr/log_ppl_diff:0.0017161983996629715 - rollout_corr/log_ppl_abs_diff:0.005773360840976238 - rollout_corr/log_ppl_diff_max:0.16158246994018555 - rollout_corr/log_ppl_diff_min:-0.061112403869628906 - rollout_corr/ppl_ratio:1.0017672777175903 - rollout_corr/chi2_token:0.0031058788299560547 - rollout_corr/chi2_seq:0.22766590118408203 - actor/pg_clipfrac:0.00047015561782851967 - actor/ppo_kl:-0.00025186119185605094 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0026446368283359334 - actor/loss:0.0026446369010955095 - actor/grad_norm:0.17974853515625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.40630865097046 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.03) - val-core/char_count/acc/mean@1:np.float64(0.03) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:50 - training/epoch:0 - critic/score/mean:0.0185546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0185546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0017351644346490502 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0017351644346490502 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.8525390625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.48046875 - response_length_non_aborted/mean:92.8525390625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.48046875 - response/aborted_ratio:0.0 - prompt_length/mean:62.734375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.11872486500215 - timing_s/old_log_prob:2.322535239996796 - timing_s/adv:0.18239247000019532 - timing_s/update_actor:4.8619464579969645 - timing_s/update_weights:1.0088089479977498 - timing_s/step:12.674584040003538 - timing_s/testing:4.8045868310000515 - timing_per_token_ms/adv:0.0011448112301592089 - timing_per_token_ms/update_actor:0.03051667048284259 - timing_per_token_ms/gen:0.043318064229469086 - perf/total_num_tokens:159321 - perf/time_per_step:12.674584040003538 - perf/throughput:12570.116660014313 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 14%|█▍ | 50/350 [11:41<1:13:37, 14.73s/it] (TaskRunnerV1 pid=2350) Training Progress: 15%|█▍ | 51/350 [11:54<1:10:50, 14.22s/it] (TaskRunnerV1 pid=2350) Training Progress: 15%|█▍ | 52/350 [12:07<1:08:49, 13.86s/it] (TaskRunnerV1 pid=2350) step:51 - global_seqlen/min:160072.0 - global_seqlen/max:160072.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160072.0 - global_seqlen/balanced_max:160072.0 - global_seqlen/mean:160072.0 - actor/entropy:1.6732419729232788 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3187282085418701 - training/rollout_probs_diff_mean:0.006765676196664572 - training/rollout_probs_diff_std:0.011169304139912128 - training/rollout_actor_probs_pearson_corr:0.9994605183601379 - rollout_corr/training_ppl:9.721668243408203 - rollout_corr/training_log_ppl:1.7308857440948486 - rollout_corr/kl:0.0015802299603819847 - rollout_corr/k3_kl:0.0016408358933404088 - rollout_corr/rollout_ppl:9.697003364562988 - rollout_corr/rollout_log_ppl:1.7292366027832031 - rollout_corr/log_ppl_diff:0.001648954814299941 - rollout_corr/log_ppl_abs_diff:0.006051151547580957 - rollout_corr/log_ppl_diff_max:0.0670311450958252 - rollout_corr/log_ppl_diff_min:-0.09648346900939941 - rollout_corr/ppl_ratio:1.0016911029815674 - rollout_corr/chi2_token:0.0034106969833374023 - rollout_corr/chi2_seq:0.42847275733947754 - actor/pg_clipfrac:0.000634824886219576 - actor/ppo_kl:1.1382731827325188e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.00643928353292722 - actor/loss:-0.006439283373765647 - actor/grad_norm:0.1971435546875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.45168685913086 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:51 - training/epoch:0 - critic/score/mean:0.0234375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0234375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.003994964994490147 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.003994964994490147 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.7109375 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.5048828125 - response_length_non_aborted/mean:94.7109375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.5048828125 - response/aborted_ratio:0.0 - prompt_length/mean:61.609375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.124503550003283 - timing_s/old_log_prob:2.3515260479980498 - timing_s/adv:0.18257610900036525 - timing_s/update_actor:5.059360560000641 - timing_s/update_weights:1.0109000859956723 - timing_s/step:12.907021173996327 - timing_per_token_ms/adv:0.0011405874169146712 - timing_per_token_ms/update_actor:0.031606780448802045 - timing_per_token_ms/gen:0.04252767002807971 - perf/total_num_tokens:160072 - perf/time_per_step:12.907021173996327 - perf/throughput:12401.932083484591 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:52 - global_seqlen/min:160190.0 - global_seqlen/max:160190.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160190.0 - global_seqlen/balanced_max:160190.0 - global_seqlen/mean:160190.0 - actor/entropy:1.653211236000061 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.33656570315361023 - training/rollout_probs_diff_mean:0.006914972327649593 - training/rollout_probs_diff_std:0.011600024998188019 - training/rollout_actor_probs_pearson_corr:0.9994245767593384 - rollout_corr/training_ppl:8.527460098266602 - rollout_corr/training_log_ppl:1.6904027462005615 - rollout_corr/kl:0.0017671207897365093 - rollout_corr/k3_kl:0.0016886991215869784 - rollout_corr/rollout_ppl:8.516420364379883 - rollout_corr/rollout_log_ppl:1.6885955333709717 - rollout_corr/log_ppl_diff:0.0018072040984407067 - rollout_corr/log_ppl_abs_diff:0.006141612771898508 - rollout_corr/log_ppl_diff_max:0.09032320976257324 - rollout_corr/log_ppl_diff_min:-0.10357999801635742 - rollout_corr/ppl_ratio:1.001850962638855 - rollout_corr/chi2_token:0.0032393932342529297 - rollout_corr/chi2_seq:0.34305882453918457 - actor/pg_clipfrac:0.0007164139327263304 - actor/ppo_kl:8.42025086361294e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.002429586718790233 - actor/loss:-0.002429586718790233 - actor/grad_norm:0.2308349609375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.385189533233643 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:52 - training/epoch:0 - critic/score/mean:0.0263671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0263671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00564020499587059 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.00564020499587059 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.779296875 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4853515625 - response_length_non_aborted/mean:93.779296875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4853515625 - response/aborted_ratio:0.0 - prompt_length/mean:62.65625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.129838099994231 - timing_s/old_log_prob:2.343008154006384 - timing_s/adv:0.18459530900145182 - timing_s/update_actor:4.852454280000529 - timing_s/update_weights:1.0149668079975527 - timing_s/step:12.697396884999762 - timing_per_token_ms/adv:0.0011523522629468246 - timing_per_token_ms/update_actor:0.03029186765716043 - timing_per_token_ms/gen:0.04300570759131762 - perf/total_num_tokens:160190 - perf/time_per_step:12.697396884999762 - perf/throughput:12615.971718521501 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:53 - global_seqlen/min:161221.0 - global_seqlen/max:161221.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161221.0 - global_seqlen/balanced_max:161221.0 - global_seqlen/mean:161221.0 - actor/entropy:1.6467338800430298 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2315123975276947 - training/rollout_probs_diff_mean:0.006806039251387119 - training/rollout_probs_diff_std:0.01123207900673151 - training/rollout_actor_probs_pearson_corr:0.9994571208953857 - rollout_corr/training_ppl:9.231858253479004 - rollout_corr/training_log_ppl:1.659082293510437 - rollout_corr/kl:0.0017698503797873855 - rollout_corr/k3_kl:0.0016055438900366426 - rollout_corr/rollout_ppl:9.2058744430542 - rollout_corr/rollout_log_ppl:1.6573617458343506 - rollout_corr/log_ppl_diff:0.0017205922631546855 - rollout_corr/log_ppl_abs_diff:0.005762444343417883 - rollout_corr/log_ppl_diff_max:0.05096101760864258 - rollout_corr/log_ppl_diff_min:-0.054993391036987305 - rollout_corr/ppl_ratio:1.0017564296722412 - rollout_corr/chi2_token:0.002890467643737793 - rollout_corr/chi2_seq:0.3023529052734375 - actor/pg_clipfrac:0.0005777210675053842 - actor/ppo_kl:-0.000273269483235102 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.006734403421432944 - actor/loss:-0.006734403898008168 - actor/grad_norm:0.2308349609375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.24596071243286 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:53 - training/epoch:0 - critic/score/mean:0.03125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.03125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.010303185321390629 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.010303185321390629 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.8486328125 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.515625 - response_length_non_aborted/mean:94.8486328125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.515625 - response/aborted_ratio:0.0 - prompt_length/mean:62.59375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.140744202006317 - timing_s/old_log_prob:2.352776833999087 - timing_s/adv:0.19931571200140752 - timing_s/update_actor:4.913959798002907 - timing_s/update_weights:1.0017071460024454 - timing_s/step:12.784215952000523 - timing_per_token_ms/adv:0.001236288771322641 - timing_per_token_ms/update_actor:0.03047965090157552 - timing_per_token_ms/gen:0.04263314493700198 - perf/total_num_tokens:161221 - perf/time_per_step:12.784215952000523 - perf/throughput:12610.941539576506 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 15%|█▌ | 53/350 [12:20<1:07:27, 13.63s/it] (TaskRunnerV1 pid=2350) Training Progress: 15%|█▌ | 54/350 [12:33<1:06:06, 13.40s/it] (TaskRunnerV1 pid=2350) Training Progress: 16%|█▌ | 55/350 [12:51<1:12:40, 14.78s/it] (TaskRunnerV1 pid=2350) step:54 - global_seqlen/min:162513.0 - global_seqlen/max:162513.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:162513.0 - global_seqlen/balanced_max:162513.0 - global_seqlen/mean:162513.0 - actor/entropy:1.6633676290512085 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3036845326423645 - training/rollout_probs_diff_mean:0.006617780774831772 - training/rollout_probs_diff_std:0.010997873730957508 - training/rollout_actor_probs_pearson_corr:0.9994871020317078 - rollout_corr/training_ppl:9.292869567871094 - rollout_corr/training_log_ppl:1.7118022441864014 - rollout_corr/kl:0.0016809806693345308 - rollout_corr/k3_kl:0.001677973661571741 - rollout_corr/rollout_ppl:9.264968872070312 - rollout_corr/rollout_log_ppl:1.7097234725952148 - rollout_corr/log_ppl_diff:0.002078640041872859 - rollout_corr/log_ppl_abs_diff:0.0059904418885707855 - rollout_corr/log_ppl_diff_max:0.0398411750793457 - rollout_corr/log_ppl_diff_min:-0.045722126960754395 - rollout_corr/ppl_ratio:1.0021158456802368 - rollout_corr/chi2_token:0.0033638477325439453 - rollout_corr/chi2_seq:0.46862292289733887 - actor/pg_clipfrac:0.0011008044518045304 - actor/ppo_kl:-0.0003146692872633139 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.006513996479043271 - actor/loss:-0.006513997039292008 - actor/grad_norm:0.2562255859375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.20088243484497 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:54 - training/epoch:0 - critic/score/mean:0.0322265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0322265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.008572215214371681 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.008572215214371681 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.7041015625 - response_length/max:128.0 - response_length/min:5.0 - response_length/clip_ratio:0.5205078125 - response_length_non_aborted/mean:94.7041015625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:5.0 - response_length_non_aborted/clip_ratio:0.5205078125 - response/aborted_ratio:0.0 - prompt_length/mean:64.0 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:4.147494559998449 - timing_s/old_log_prob:2.3575860560013098 - timing_s/adv:0.18180371000198647 - timing_s/update_actor:4.858538842003327 - timing_s/update_weights:1.0073345990022062 - timing_s/step:12.74298898899724 - timing_per_token_ms/adv:0.0011187025653454584 - timing_per_token_ms/update_actor:0.029896308861465402 - timing_per_token_ms/gen:0.04276781669878888 - perf/total_num_tokens:162513 - perf/time_per_step:12.74298898899724 - perf/throughput:12753.130379404678 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:55 - global_seqlen/min:157507.0 - global_seqlen/max:157507.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157507.0 - global_seqlen/balanced_max:157507.0 - global_seqlen/mean:157507.0 - actor/entropy:1.6967037916183472 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.33788609504699707 - training/rollout_probs_diff_mean:0.006690599024295807 - training/rollout_probs_diff_std:0.01121005043387413 - training/rollout_actor_probs_pearson_corr:0.9994669556617737 - rollout_corr/training_ppl:17.779064178466797 - rollout_corr/training_log_ppl:1.7774230241775513 - rollout_corr/kl:0.0017590412171557546 - rollout_corr/k3_kl:0.001696460647508502 - rollout_corr/rollout_ppl:17.906696319580078 - rollout_corr/rollout_log_ppl:1.7757019996643066 - rollout_corr/log_ppl_diff:0.0017210771329700947 - rollout_corr/log_ppl_abs_diff:0.006618658546358347 - rollout_corr/log_ppl_diff_max:0.06245231628417969 - rollout_corr/log_ppl_diff_min:-0.10167360305786133 - rollout_corr/ppl_ratio:1.0017764568328857 - rollout_corr/chi2_token:0.003259897232055664 - rollout_corr/chi2_seq:0.3999723196029663 - actor/pg_clipfrac:0.0007803892234126881 - actor/ppo_kl:-0.00012372534176089783 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.005461878896312555 - actor/loss:0.005461878783535212 - actor/grad_norm:0.25 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.19196319580078 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.036) - val-core/char_count/acc/mean@1:np.float64(0.036) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:55 - training/epoch:0 - critic/score/mean:0.0322265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0322265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005569749046117067 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.005569749046117067 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.2373046875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.494140625 - response_length_non_aborted/mean:92.2373046875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.494140625 - response/aborted_ratio:0.0 - prompt_length/mean:61.578125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.136349054002494 - timing_s/old_log_prob:2.582423251005821 - timing_s/adv:0.18198907800251618 - timing_s/update_actor:4.833376632996078 - timing_s/update_weights:1.0128071119979722 - timing_s/step:12.924817920997157 - timing_s/testing:4.9486640989998705 - timing_per_token_ms/adv:0.0011554348568794796 - timing_per_token_ms/update_actor:0.030686741751135366 - timing_per_token_ms/gen:0.0437935972515113 - perf/total_num_tokens:157507 - perf/time_per_step:12.924817920997157 - perf/throughput:12186.399913929949 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:56 - global_seqlen/min:158645.0 - global_seqlen/max:158645.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158645.0 - global_seqlen/balanced_max:158645.0 - global_seqlen/mean:158645.0 - actor/entropy:1.6853628158569336 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.23016422986984253 - training/rollout_probs_diff_mean:0.00681348517537117 - training/rollout_probs_diff_std:0.0112074576318264 - training/rollout_actor_probs_pearson_corr:0.9994577169418335 - rollout_corr/training_ppl:11.318178176879883 - rollout_corr/training_log_ppl:1.740752100944519 - rollout_corr/kl:0.001561740762554109 - rollout_corr/k3_kl:0.0016883468488231301 - rollout_corr/rollout_ppl:11.238391876220703 - rollout_corr/rollout_log_ppl:1.7388328313827515 - rollout_corr/log_ppl_diff:0.0019193798070773482 - rollout_corr/log_ppl_abs_diff:0.005908200982958078 - rollout_corr/log_ppl_diff_max:0.12091255187988281 - rollout_corr/log_ppl_diff_min:-0.04360365867614746 - rollout_corr/ppl_ratio:1.0019673109054565 - rollout_corr/chi2_token:0.0036478042602539062 - rollout_corr/chi2_seq:0.30397605895996094 - actor/pg_clipfrac:0.000962039087729257 - actor/ppo_kl:-7.784154962564573e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0065827471553348005 - actor/loss:-0.006582746864296496 - actor/grad_norm:0.2490234375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.466435432434082 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:56 - training/epoch:0 - critic/score/mean:0.037109375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.037109375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0018211599672213197 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.0018211599672213197 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.0205078125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5068359375 - response_length_non_aborted/mean:93.0205078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5068359375 - response/aborted_ratio:0.0 - prompt_length/mean:61.90625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.148208799000713 - timing_s/old_log_prob:2.354699850999168 - timing_s/adv:0.19080611499521183 - timing_s/update_actor:4.840620686001785 - timing_s/update_weights:1.012182631995529 - timing_s/step:12.727283038999303 - timing_per_token_ms/adv:0.001202723785780906 - timing_per_token_ms/update_actor:0.030512280160117145 - timing_per_token_ms/gen:0.04354937691202076 - perf/total_num_tokens:158645 - perf/time_per_step:12.727283038999303 - perf/throughput:12464.95418652005 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 16%|█▌ | 56/350 [13:04<1:09:54, 14.27s/it] (TaskRunnerV1 pid=2350) Training Progress: 16%|█▋ | 57/350 [13:17<1:07:42, 13.87s/it] (TaskRunnerV1 pid=2350) Training Progress: 17%|█▋ | 58/350 [13:29<1:06:04, 13.58s/it] (TaskRunnerV1 pid=2350) step:57 - global_seqlen/min:160077.0 - global_seqlen/max:160077.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160077.0 - global_seqlen/balanced_max:160077.0 - global_seqlen/mean:160077.0 - actor/entropy:1.6924207210540771 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.29203587770462036 - training/rollout_probs_diff_mean:0.006821483373641968 - training/rollout_probs_diff_std:0.011191950179636478 - training/rollout_actor_probs_pearson_corr:0.9994539022445679 - rollout_corr/training_ppl:9.629927635192871 - rollout_corr/training_log_ppl:1.7574007511138916 - rollout_corr/kl:0.002159254625439644 - rollout_corr/k3_kl:0.0017117641400545835 - rollout_corr/rollout_ppl:9.655108451843262 - rollout_corr/rollout_log_ppl:1.755342960357666 - rollout_corr/log_ppl_diff:0.0020578771363943815 - rollout_corr/log_ppl_abs_diff:0.006335465237498283 - rollout_corr/log_ppl_diff_max:0.07898950576782227 - rollout_corr/log_ppl_diff_min:-0.13831853866577148 - rollout_corr/ppl_ratio:1.00211763381958 - rollout_corr/chi2_token:0.0025159120559692383 - rollout_corr/chi2_seq:0.24840545654296875 - actor/pg_clipfrac:0.0005538507515060095 - actor/ppo_kl:-0.00044219247138244105 - actor/pg_clipfrac_lower:1.1760278236276159e-05 - actor/pg_loss:-0.004941020524711348 - actor/loss:-0.004941020626574755 - actor/grad_norm:0.2288818359375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.407204627990723 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:57 - training/epoch:0 - critic/score/mean:0.03125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.03125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.008602100424468517 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.008602100424468517 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.0283203125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.51171875 - response_length_non_aborted/mean:94.0283203125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.51171875 - response/aborted_ratio:0.0 - prompt_length/mean:62.296875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:4.138566331996117 - timing_s/old_log_prob:2.3420123909963877 - timing_s/adv:0.22542239400354447 - timing_s/update_actor:4.865699229994789 - timing_s/update_weights:1.045998582005268 - timing_s/step:12.803833801997826 - timing_per_token_ms/adv:0.001408212260371849 - timing_per_token_ms/update_actor:0.03039599211626148 - timing_per_token_ms/gen:0.04298246177489865 - perf/total_num_tokens:160077 - perf/time_per_step:12.803833801997826 - perf/throughput:12502.270997536898 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:58 - global_seqlen/min:156073.0 - global_seqlen/max:156073.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156073.0 - global_seqlen/balanced_max:156073.0 - global_seqlen/mean:156073.0 - actor/entropy:1.6747231483459473 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.28477171063423157 - training/rollout_probs_diff_mean:0.006793759297579527 - training/rollout_probs_diff_std:0.011179654859006405 - training/rollout_actor_probs_pearson_corr:0.9994581341743469 - rollout_corr/training_ppl:9.07621955871582 - rollout_corr/training_log_ppl:1.7180829048156738 - rollout_corr/kl:0.0016629908932372928 - rollout_corr/k3_kl:0.0016484371153637767 - rollout_corr/rollout_ppl:9.06005573272705 - rollout_corr/rollout_log_ppl:1.7164373397827148 - rollout_corr/log_ppl_diff:0.0016455852892249823 - rollout_corr/log_ppl_abs_diff:0.006231799256056547 - rollout_corr/log_ppl_diff_max:0.10518956184387207 - rollout_corr/log_ppl_diff_min:-0.11383438110351562 - rollout_corr/ppl_ratio:1.0016958713531494 - rollout_corr/chi2_token:0.0032657384872436523 - rollout_corr/chi2_seq:0.3289254903793335 - actor/pg_clipfrac:0.0007356000866517813 - actor/ppo_kl:-0.00015430981301278734 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0039044337754603475 - actor/loss:-0.0039044334553182125 - actor/grad_norm:0.2479248046875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.415379524230957 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:58 - training/epoch:0 - critic/score/mean:0.03515625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.03515625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.003623481374233961 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.003623481374233961 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:92.2431640625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4912109375 - response_length_non_aborted/mean:92.2431640625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4912109375 - response/aborted_ratio:0.0 - prompt_length/mean:60.171875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.130220238999755 - timing_s/old_log_prob:2.537485435997951 - timing_s/adv:0.1831452789992909 - timing_s/update_actor:4.724805021003704 - timing_s/update_weights:1.0191495230028522 - timing_s/step:12.77821440200205 - timing_per_token_ms/adv:0.0011734590800413325 - timing_per_token_ms/update_actor:0.030273045440298473 - timing_per_token_ms/gen:0.04372593073038266 - perf/total_num_tokens:156073 - perf/time_per_step:12.77821440200205 - perf/throughput:12213.991336344066 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:59 - global_seqlen/min:161128.0 - global_seqlen/max:161128.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161128.0 - global_seqlen/balanced_max:161128.0 - global_seqlen/mean:161128.0 - actor/entropy:1.731061577796936 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.28732311725616455 - training/rollout_probs_diff_mean:0.006781057920306921 - training/rollout_probs_diff_std:0.011190822347998619 - training/rollout_actor_probs_pearson_corr:0.999459981918335 - rollout_corr/training_ppl:10.578409194946289 - rollout_corr/training_log_ppl:1.7950899600982666 - rollout_corr/kl:0.0017401318764314055 - rollout_corr/k3_kl:0.0016753996023908257 - rollout_corr/rollout_ppl:10.53271484375 - rollout_corr/rollout_log_ppl:1.7933974266052246 - rollout_corr/log_ppl_diff:0.001692681573331356 - rollout_corr/log_ppl_abs_diff:0.006249530240893364 - rollout_corr/log_ppl_diff_max:0.06946253776550293 - rollout_corr/log_ppl_diff_min:-0.0423356294631958 - rollout_corr/ppl_ratio:1.0017362833023071 - rollout_corr/chi2_token:0.0032378435134887695 - rollout_corr/chi2_seq:0.45935678482055664 - actor/pg_clipfrac:0.0007325472336863943 - actor/ppo_kl:0.00019208372668799714 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.008473590802168474 - actor/loss:0.008473590249195695 - actor/grad_norm:0.2174072265625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.392744064331055 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:59 - training/epoch:0 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005029091611504555 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:-0.005029091611504555 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.8671875 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.5205078125 - response_length_non_aborted/mean:93.8671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.5205078125 - response/aborted_ratio:0.0 - prompt_length/mean:63.484375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.126448490002076 - timing_s/old_log_prob:2.5441495699997176 - timing_s/adv:0.24472101999708684 - timing_s/update_actor:4.865464321002946 - timing_s/update_weights:1.014990730000136 - timing_s/step:12.973767729003157 - timing_per_token_ms/adv:0.0015187988431376721 - timing_per_token_ms/update_actor:0.03019626831465013 - timing_per_token_ms/gen:0.04293017571787428 - perf/total_num_tokens:161128 - perf/time_per_step:12.973767729003157 - perf/throughput:12419.522483032792 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 17%|█▋ | 59/350 [13:43<1:05:09, 13.44s/it] (TaskRunnerV1 pid=2350) Training Progress: 17%|█▋ | 60/350 [14:00<1:11:11, 14.73s/it] (TransferQueueController pid=2570) 2026-11-01 17:04:09,817 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_ebda4368: [Performance] Total success requests: 112777, Total req/min: 22547.50, Total avg process time: 0.0001s; (TransferQueueController pid=2570) Time range: last 5.00 minutes; (TransferQueueController pid=2570) Per-operation statistics: KV_RETRIEVE_META: req_count=42836, req/min=8564.20, avg_time=0.000258s, max_time=0.070920s, min_time=0.000063s; CLEAR_META: req_count=53, req/min=10.60, avg_time=0.001885s, max_time=0.003818s, min_time=0.000948s; SET_CUSTOM_META: req_count=42688, req/min=8534.61, avg_time=0.000010s, max_time=0.000304s, min_time=0.000003s; KV_LIST: req_count=80, req/min=15.99, avg_time=0.000349s, max_time=0.001020s, min_time=0.000127s; NOTIFY_DATA_UPDATE: req_count=27099, req/min=5417.90, avg_time=0.000088s, max_time=0.001473s, min_time=0.000030s; KV_RETRIEVE_KEYS: req_count=21, req/min=4.20, avg_time=0.000150s, max_time=0.000232s, min_time=0.000106s (SimpleStorageUnit pid=2676) Per-operation statistics: CLEAR_DATA: req_count=53, req/min=10.60, avg_time=0.001384s, max_time=0.004319s, min_time=0.000371s; PUT_DATA: req_count=3457, req/min=691.14, avg_time=0.000602s, max_time=0.002720s, min_time=0.000311s; GET_DATA: req_count=115, req/min=22.99, avg_time=0.000632s, max_time=0.001250s, min_time=0.000374s (TaskRunnerV1 pid=2350) Training Progress: 17%|█▋ | 61/350 [14:13<1:08:21, 14.19s/it] (SimpleStorageUnit pid=2691) 2026-11-01 17:04:09,832 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_52739293: [Performance] Total success requests: 3613, Total req/min: 722.33, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2691) Time range: last 5.00 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2691) Per-operation statistics: CLEAR_DATA: req_count=53, req/min=10.60, avg_time=0.001495s, max_time=0.004320s, min_time=0.000406s; PUT_DATA: req_count=3445, req/min=688.75, avg_time=0.000598s, max_time=0.002935s, min_time=0.000314s; GET_DATA: req_count=115, req/min=22.99, avg_time=0.000633s, max_time=0.001252s, min_time=0.000375s [repeated 7x across cluster] (TaskRunnerV1 pid=2350) step:60 - global_seqlen/min:159112.0 - global_seqlen/max:159112.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159112.0 - global_seqlen/balanced_max:159112.0 - global_seqlen/mean:159112.0 - actor/entropy:1.6926518678665161 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3083854615688324 - training/rollout_probs_diff_mean:0.006902615074068308 - training/rollout_probs_diff_std:0.011385306715965271 - training/rollout_actor_probs_pearson_corr:0.9994407296180725 - rollout_corr/training_ppl:21.80023765563965 - rollout_corr/training_log_ppl:1.7564581632614136 - rollout_corr/kl:0.0015623975777998567 - rollout_corr/k3_kl:0.001721833716146648 - rollout_corr/rollout_ppl:20.847930908203125 - rollout_corr/rollout_log_ppl:1.754512906074524 - rollout_corr/log_ppl_diff:0.0019452739506959915 - rollout_corr/log_ppl_abs_diff:0.006346499547362328 - rollout_corr/log_ppl_diff_max:0.08231830596923828 - rollout_corr/log_ppl_diff_min:-0.09724211692810059 - rollout_corr/ppl_ratio:1.0019935369491577 - rollout_corr/chi2_token:0.003786444664001465 - rollout_corr/chi2_seq:0.41576266288757324 - actor/pg_clipfrac:0.0006379224602521087 - actor/ppo_kl:-0.00013231494854911338 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0027658142207656056 - actor/loss:-0.002765814191661775 - actor/grad_norm:0.201416015625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.41732120513916 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.033) - val-core/char_count/acc/mean@1:np.float64(0.033) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:60 - training/epoch:0 - critic/score/mean:0.021484375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.021484375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0049485559575259686 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.0049485559575259686 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:92.7421875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.501953125 - response_length_non_aborted/mean:92.7421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.501953125 - response/aborted_ratio:0.0 - prompt_length/mean:62.640625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.123211484999047 - timing_s/old_log_prob:2.3895795549979084 - timing_s/adv:0.39059183400240727 - timing_s/update_actor:4.882251600000018 - timing_s/update_weights:1.0222364879955421 - timing_s/step:12.981485370997689 - timing_s/testing:4.628110459998425 - timing_per_token_ms/adv:0.0024548232314495905 - timing_per_token_ms/update_actor:0.030684370757705377 - timing_per_token_ms/gen:0.04341685078130577 - perf/total_num_tokens:159112 - perf/time_per_step:12.981485370997689 - perf/throughput:12256.840835446821 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:61 - global_seqlen/min:157303.0 - global_seqlen/max:157303.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157303.0 - global_seqlen/balanced_max:157303.0 - global_seqlen/mean:157303.0 - actor/entropy:1.6961264610290527 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3924362063407898 - training/rollout_probs_diff_mean:0.006796572357416153 - training/rollout_probs_diff_std:0.011413122527301311 - training/rollout_actor_probs_pearson_corr:0.9994454979896545 - rollout_corr/training_ppl:9.32843017578125 - rollout_corr/training_log_ppl:1.7359501123428345 - rollout_corr/kl:0.001671305508352816 - rollout_corr/k3_kl:0.0016953048761934042 - rollout_corr/rollout_ppl:9.300490379333496 - rollout_corr/rollout_log_ppl:1.7337045669555664 - rollout_corr/log_ppl_diff:0.0022455144207924604 - rollout_corr/log_ppl_abs_diff:0.006121929734945297 - rollout_corr/log_ppl_diff_max:0.0535578727722168 - rollout_corr/log_ppl_diff_min:-0.05464053153991699 - rollout_corr/ppl_ratio:1.00228750705719 - rollout_corr/chi2_token:0.003471851348876953 - rollout_corr/chi2_seq:0.36053192615509033 - actor/pg_clipfrac:0.0004674982283177087 - actor/ppo_kl:-0.00011664855215561451 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0029141288832761347 - actor/loss:0.002914128650445491 - actor/grad_norm:0.17437744140625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.606499671936035 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:61 - training/epoch:0 - critic/score/mean:0.0224609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0224609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.003078756621107459 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.003078756621107459 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.3037109375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4853515625 - response_length_non_aborted/mean:92.3037109375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4853515625 - response/aborted_ratio:0.0 - prompt_length/mean:61.3125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.150861672998872 - timing_s/old_log_prob:2.343198656999448 - timing_s/adv:0.19381540099857375 - timing_s/update_actor:4.743806136997591 - timing_s/update_weights:1.0053107689964236 - timing_s/step:12.814977807996911 - timing_per_token_ms/adv:0.0012321150963336602 - timing_per_token_ms/update_actor:0.030157124384135023 - timing_per_token_ms/gen:0.043915632550057364 - perf/total_num_tokens:157303 - perf/time_per_step:12.814977807996911 - perf/throughput:12274.933469009868 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:62 - global_seqlen/min:158001.0 - global_seqlen/max:158001.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158001.0 - global_seqlen/balanced_max:158001.0 - global_seqlen/mean:158001.0 - actor/entropy:1.6812398433685303 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3880069851875305 - training/rollout_probs_diff_mean:0.006802965421229601 - training/rollout_probs_diff_std:0.011298883706331253 - training/rollout_actor_probs_pearson_corr:0.9994475245475769 - rollout_corr/training_ppl:9.218596458435059 - rollout_corr/training_log_ppl:1.7264655828475952 - rollout_corr/kl:0.001446177251636982 - rollout_corr/k3_kl:0.0016493427101522684 - rollout_corr/rollout_ppl:9.145868301391602 - rollout_corr/rollout_log_ppl:1.725010871887207 - rollout_corr/log_ppl_diff:0.0014546590391546488 - rollout_corr/log_ppl_abs_diff:0.005852645728737116 - rollout_corr/log_ppl_diff_max:0.1291193962097168 - rollout_corr/log_ppl_diff_min:-0.04113030433654785 - rollout_corr/ppl_ratio:1.001499891281128 - rollout_corr/chi2_token:0.0037190914154052734 - rollout_corr/chi2_seq:0.37773191928863525 - actor/pg_clipfrac:0.0008289570644895623 - actor/ppo_kl:-0.00017074155219631848 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0005632913671433926 - actor/loss:-0.0005632914253510535 - actor/grad_norm:0.244873046875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.481205463409424 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:62 - training/epoch:0 - critic/score/mean:0.0341796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0341796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00032467825803905725 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:-0.00032467825803905725 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.2353515625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5048828125 - response_length_non_aborted/mean:93.2353515625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5048828125 - response/aborted_ratio:0.0 - prompt_length/mean:61.0625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.128627082995081 - timing_s/old_log_prob:2.546613703998446 - timing_s/adv:0.18374616499932017 - timing_s/update_actor:4.792728005995741 - timing_s/update_weights:1.0132649799998035 - timing_s/step:12.835753798004589 - timing_per_token_ms/adv:0.0011629430509890455 - timing_per_token_ms/update_actor:0.030333529572570687 - timing_per_token_ms/gen:0.04324392323478974 - perf/total_num_tokens:158001 - perf/time_per_step:12.835753798004589 - perf/throughput:12309.44457851493 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 18%|█▊ | 62/350 [14:26<1:06:21, 13.83s/it] (TaskRunnerV1 pid=2350) Training Progress: 18%|█▊ | 63/350 [14:39<1:05:01, 13.60s/it] (TaskRunnerV1 pid=2350) Training Progress: 18%|█▊ | 64/350 [14:52<1:03:58, 13.42s/it] (TaskRunnerV1 pid=2350) step:63 - global_seqlen/min:155118.0 - global_seqlen/max:155118.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155118.0 - global_seqlen/balanced_max:155118.0 - global_seqlen/mean:155118.0 - actor/entropy:1.743140697479248 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2769882082939148 - training/rollout_probs_diff_mean:0.006943186745047569 - training/rollout_probs_diff_std:0.011497078463435173 - training/rollout_actor_probs_pearson_corr:0.9994289875030518 - rollout_corr/training_ppl:15.38906192779541 - rollout_corr/training_log_ppl:1.8465113639831543 - rollout_corr/kl:0.0016818847507238388 - rollout_corr/k3_kl:0.0017928635934367776 - rollout_corr/rollout_ppl:14.978361129760742 - rollout_corr/rollout_log_ppl:1.8445279598236084 - rollout_corr/log_ppl_diff:0.0019834255799651146 - rollout_corr/log_ppl_abs_diff:0.006762479431927204 - rollout_corr/log_ppl_diff_max:0.11465120315551758 - rollout_corr/log_ppl_diff_min:-0.05946922302246094 - rollout_corr/ppl_ratio:1.0020437240600586 - rollout_corr/chi2_token:0.0038563013076782227 - rollout_corr/chi2_seq:0.4461047649383545 - actor/pg_clipfrac:0.0008322740062188435 - actor/ppo_kl:0.00020104546132415635 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0025395989505341277 - actor/loss:-0.0025395989941898733 - actor/grad_norm:0.2569580078125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.491904258728027 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:63 - training/epoch:0 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0020150772761553526 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0020150772761553526 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:89.560546875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.439453125 - response_length_non_aborted/mean:89.560546875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.439453125 - response/aborted_ratio:0.0 - prompt_length/mean:61.921875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.150525012002618 - timing_s/old_log_prob:2.2903327260064543 - timing_s/adv:0.38292915299825836 - timing_s/update_actor:4.833506977003708 - timing_s/update_weights:1.0249366269999882 - timing_s/step:12.869876255004783 - timing_per_token_ms/adv:0.0024686313193714358 - timing_per_token_ms/update_actor:0.03116019402650697 - timing_per_token_ms/gen:0.045257060429643635 - perf/total_num_tokens:155118 - perf/time_per_step:12.869876255004783 - perf/throughput:12052.796540268082 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:64 - global_seqlen/min:155762.0 - global_seqlen/max:155762.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155762.0 - global_seqlen/balanced_max:155762.0 - global_seqlen/mean:155762.0 - actor/entropy:1.695665955543518 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.24348986148834229 - training/rollout_probs_diff_mean:0.0067901345901191235 - training/rollout_probs_diff_std:0.011360043659806252 - training/rollout_actor_probs_pearson_corr:0.9994463920593262 - rollout_corr/training_ppl:9.516260147094727 - rollout_corr/training_log_ppl:1.73301362991333 - rollout_corr/kl:0.0019752047955989838 - rollout_corr/k3_kl:0.0016596902860328555 - rollout_corr/rollout_ppl:9.471940994262695 - rollout_corr/rollout_log_ppl:1.730352759361267 - rollout_corr/log_ppl_diff:0.0026609927881509066 - rollout_corr/log_ppl_abs_diff:0.006315065547823906 - rollout_corr/log_ppl_diff_max:0.09171724319458008 - rollout_corr/log_ppl_diff_min:-0.059833645820617676 - rollout_corr/ppl_ratio:1.0027081966400146 - rollout_corr/chi2_token:0.002682328224182129 - rollout_corr/chi2_seq:0.3318418264389038 - actor/pg_clipfrac:0.0006806707272820252 - actor/ppo_kl:-0.0003106082125062434 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0055256579635170056 - actor/loss:0.005525656510144472 - actor/grad_norm:0.23583984375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.429961681365967 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:64 - training/epoch:0 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004139031283557415 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:-0.004139031283557415 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:90.533203125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.46875 - response_length_non_aborted/mean:90.533203125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.46875 - response/aborted_ratio:0.0 - prompt_length/mean:61.578125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.146089481997478 - timing_s/old_log_prob:2.3196681959962007 - timing_s/adv:0.3880224599997746 - timing_s/update_actor:4.84397108900157 - timing_s/update_weights:1.0038290759985102 - timing_s/step:12.878774024997256 - timing_per_token_ms/adv:0.0024911240225457725 - timing_per_token_ms/update_actor:0.03109854193578389 - timing_per_token_ms/gen:0.044722989687803145 - perf/total_num_tokens:155762 - perf/time_per_step:12.878774024997256 - perf/throughput:12094.474186570193 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:65 - global_seqlen/min:158325.0 - global_seqlen/max:158325.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158325.0 - global_seqlen/balanced_max:158325.0 - global_seqlen/mean:158325.0 - actor/entropy:1.7336127758026123 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3933667540550232 - training/rollout_probs_diff_mean:0.006895245984196663 - training/rollout_probs_diff_std:0.011439621448516846 - training/rollout_actor_probs_pearson_corr:0.999433159828186 - rollout_corr/training_ppl:10.915348052978516 - rollout_corr/training_log_ppl:1.7832310199737549 - rollout_corr/kl:0.0019825580529868603 - rollout_corr/k3_kl:0.0016727359034121037 - rollout_corr/rollout_ppl:10.864143371582031 - rollout_corr/rollout_log_ppl:1.7809388637542725 - rollout_corr/log_ppl_diff:0.002292273798957467 - rollout_corr/log_ppl_abs_diff:0.006100997794419527 - rollout_corr/log_ppl_diff_max:0.08446168899536133 - rollout_corr/log_ppl_diff_min:-0.050237178802490234 - rollout_corr/ppl_ratio:1.002333641052246 - rollout_corr/chi2_token:0.0027244091033935547 - rollout_corr/chi2_seq:0.22339117527008057 - actor/pg_clipfrac:0.0005645604845388637 - actor/ppo_kl:-0.0004989772158978667 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005310610261403781 - actor/loss:-0.005310610169544816 - actor/grad_norm:0.193603515625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.50630760192871 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.028) - val-core/char_count/acc/mean@1:np.float64(0.028) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:65 - training/epoch:0 - critic/score/mean:0.0224609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0224609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.01091381162405014 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.01091381162405014 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:92.7861328125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.509765625 - response_length_non_aborted/mean:92.7861328125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.509765625 - response/aborted_ratio:0.0 - prompt_length/mean:61.828125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.126999487998546 - timing_s/old_log_prob:2.3425454719981644 - timing_s/adv:0.2359260150042246 - timing_s/update_actor:5.039266940999369 - timing_s/update_weights:1.0222656139958417 - timing_s/step:12.941228910000063 - timing_s/testing:4.654607621007017 - timing_per_token_ms/adv:0.0014901374704198618 - timing_per_token_ms/update_actor:0.03182862429180085 - timing_per_token_ms/gen:0.04343615597864025 - perf/total_num_tokens:158325 - perf/time_per_step:12.941228910000063 - perf/throughput:12234.154970990248 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 19%|█▊ | 65/350 [15:10<1:09:52, 14.71s/it] (TaskRunnerV1 pid=2350) Training Progress: 19%|█▉ | 66/350 [15:23<1:07:15, 14.21s/it] (TaskRunnerV1 pid=2350) Training Progress: 19%|█▉ | 67/350 [15:36<1:05:28, 13.88s/it] (TaskRunnerV1 pid=2350) step:66 - global_seqlen/min:158789.0 - global_seqlen/max:158789.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158789.0 - global_seqlen/balanced_max:158789.0 - global_seqlen/mean:158789.0 - actor/entropy:1.6637988090515137 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.33328086137771606 - training/rollout_probs_diff_mean:0.006834408268332481 - training/rollout_probs_diff_std:0.01129087246954441 - training/rollout_actor_probs_pearson_corr:0.999448299407959 - rollout_corr/training_ppl:11.25766658782959 - rollout_corr/training_log_ppl:1.7515004873275757 - rollout_corr/kl:0.0015508520882576704 - rollout_corr/k3_kl:0.0016871426487341523 - rollout_corr/rollout_ppl:11.225486755371094 - rollout_corr/rollout_log_ppl:1.7498680353164673 - rollout_corr/log_ppl_diff:0.0016322641167789698 - rollout_corr/log_ppl_abs_diff:0.006317250896245241 - rollout_corr/log_ppl_diff_max:0.14253497123718262 - rollout_corr/log_ppl_diff_min:-0.05735063552856445 - rollout_corr/ppl_ratio:1.001684308052063 - rollout_corr/chi2_token:0.003747105598449707 - rollout_corr/chi2_seq:0.5181328058242798 - actor/pg_clipfrac:0.0010216051244545572 - actor/ppo_kl:0.00020196223249513423 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005181082982744556 - actor/loss:-0.005181083222851157 - actor/grad_norm:0.268798828125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.630154609680176 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:66 - training/epoch:0 - critic/score/mean:0.03515625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.03515625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.004153733607381582 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.004153733607381582 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:92.1923828125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4931640625 - response_length_non_aborted/mean:92.1923828125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4931640625 - response/aborted_ratio:0.0 - prompt_length/mean:62.875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.131653559001279 - timing_s/old_log_prob:2.3450043150005513 - timing_s/adv:0.40041245800239267 - timing_s/update_actor:4.840106921001279 - timing_s/update_weights:1.011192438993021 - timing_s/step:12.904573448002338 - timing_per_token_ms/adv:0.002521663704679749 - timing_per_token_ms/update_actor:0.030481374156908093 - timing_per_token_ms/gen:0.04376519844289263 - perf/total_num_tokens:158789 - perf/time_per_step:12.904573448002338 - perf/throughput:12304.862352856842 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:67 - global_seqlen/min:159423.0 - global_seqlen/max:159423.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159423.0 - global_seqlen/balanced_max:159423.0 - global_seqlen/mean:159423.0 - actor/entropy:1.7040809392929077 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.20368877053260803 - training/rollout_probs_diff_mean:0.006598299834877253 - training/rollout_probs_diff_std:0.01089486200362444 - training/rollout_actor_probs_pearson_corr:0.9994915127754211 - rollout_corr/training_ppl:11.142366409301758 - rollout_corr/training_log_ppl:1.78816556930542 - rollout_corr/kl:0.0016557087656110525 - rollout_corr/k3_kl:0.001663647242821753 - rollout_corr/rollout_ppl:11.111167907714844 - rollout_corr/rollout_log_ppl:1.7859877347946167 - rollout_corr/log_ppl_diff:0.0021778333466500044 - rollout_corr/log_ppl_abs_diff:0.00616684602573514 - rollout_corr/log_ppl_diff_max:0.11249279975891113 - rollout_corr/log_ppl_diff_min:-0.06552696228027344 - rollout_corr/ppl_ratio:1.002229928970337 - rollout_corr/chi2_token:0.003359079360961914 - rollout_corr/chi2_seq:0.2487102746963501 - actor/pg_clipfrac:0.000591411968950221 - actor/ppo_kl:1.7428760202164995e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.006405616499250755 - actor/loss:-0.00640561617910862 - actor/grad_norm:0.2012939453125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.659369468688965 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:67 - training/epoch:0 - critic/score/mean:0.0205078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0205078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.005278305150568485 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.005278305150568485 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.9990234375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5009765625 - response_length_non_aborted/mean:92.9990234375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5009765625 - response/aborted_ratio:0.0 - prompt_length/mean:62.6875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0234375 - timing_s/gen:4.145586749997165 - timing_s/old_log_prob:2.3376734370031045 - timing_s/adv:0.38437904699821956 - timing_s/update_actor:4.86686137200013 - timing_s/update_weights:1.0170768089956255 - timing_s/step:12.922813060999033 - timing_per_token_ms/adv:0.0024110639430836175 - timing_per_token_ms/update_actor:0.03052797508515164 - timing_per_token_ms/gen:0.04353190400181836 - perf/total_num_tokens:159423 - perf/time_per_step:12.922813060999033 - perf/throughput:12336.555457970493 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:68 - global_seqlen/min:158797.0 - global_seqlen/max:158797.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158797.0 - global_seqlen/balanced_max:158797.0 - global_seqlen/mean:158797.0 - actor/entropy:1.7131578922271729 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3178353011608124 - training/rollout_probs_diff_mean:0.006764136720448732 - training/rollout_probs_diff_std:0.011275691911578178 - training/rollout_actor_probs_pearson_corr:0.9994556903839111 - rollout_corr/training_ppl:11.322396278381348 - rollout_corr/training_log_ppl:1.7772293090820312 - rollout_corr/kl:0.001445320202037692 - rollout_corr/k3_kl:0.0016518934862688184 - rollout_corr/rollout_ppl:11.202537536621094 - rollout_corr/rollout_log_ppl:1.775615930557251 - rollout_corr/log_ppl_diff:0.001613469561561942 - rollout_corr/log_ppl_abs_diff:0.006153975613415241 - rollout_corr/log_ppl_diff_max:0.12516069412231445 - rollout_corr/log_ppl_diff_min:-0.08021402359008789 - rollout_corr/ppl_ratio:1.0016591548919678 - rollout_corr/chi2_token:0.0037442445755004883 - rollout_corr/chi2_seq:0.5485594272613525 - actor/pg_clipfrac:0.0008296516061818693 - actor/ppo_kl:-4.016125118747975e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0005208237853366882 - actor/loss:-0.0005208236398175359 - actor/grad_norm:0.2271728515625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.664966106414795 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:68 - training/epoch:0 - critic/score/mean:0.0322265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0322265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00016370204684790224 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.935412585735321 - critic/returns/mean:-0.00016370204684790224 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.935412585735321 - response_length/mean:94.1533203125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5087890625 - response_length_non_aborted/mean:94.1533203125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5087890625 - response/aborted_ratio:0.0 - prompt_length/mean:60.921875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.14114493499801 - timing_s/old_log_prob:2.357224193001457 - timing_s/adv:0.1834627290008939 - timing_s/update_actor:5.057685596999363 - timing_s/update_weights:1.0078463300014846 - timing_s/step:12.925717170001008 - timing_per_token_ms/adv:0.0011553286837968845 - timing_per_token_ms/update_actor:0.03185000722305436 - timing_per_token_ms/gen:0.04295214270895015 - perf/total_num_tokens:158797 - perf/time_per_step:12.925717170001008 - perf/throughput:12285.353138358018 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 19%|█▉ | 68/350 [15:49<1:04:05, 13.64s/it] (TaskRunnerV1 pid=2350) Training Progress: 20%|█▉ | 69/350 [16:02<1:03:06, 13.47s/it] (TaskRunnerV1 pid=2350) Training Progress: 20%|██ | 70/350 [16:20<1:09:01, 14.79s/it] (TaskRunnerV1 pid=2350) step:69 - global_seqlen/min:161911.0 - global_seqlen/max:161911.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161911.0 - global_seqlen/balanced_max:161911.0 - global_seqlen/mean:161911.0 - actor/entropy:1.6445554494857788 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2932818531990051 - training/rollout_probs_diff_mean:0.006769736763089895 - training/rollout_probs_diff_std:0.011287443339824677 - training/rollout_actor_probs_pearson_corr:0.9994518756866455 - rollout_corr/training_ppl:8.755459785461426 - rollout_corr/training_log_ppl:1.6841353178024292 - rollout_corr/kl:0.0018479969585314393 - rollout_corr/k3_kl:0.001640685717575252 - rollout_corr/rollout_ppl:8.737035751342773 - rollout_corr/rollout_log_ppl:1.6825273036956787 - rollout_corr/log_ppl_diff:0.0016080554341897368 - rollout_corr/log_ppl_abs_diff:0.005785027984529734 - rollout_corr/log_ppl_diff_max:0.04532670974731445 - rollout_corr/log_ppl_diff_min:-0.04644644260406494 - rollout_corr/ppl_ratio:1.0016430616378784 - rollout_corr/chi2_token:0.0028748512268066406 - rollout_corr/chi2_seq:0.30056941509246826 - actor/pg_clipfrac:0.0010722806825166724 - actor/ppo_kl:-0.00034586497470930533 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.004220900031214114 - actor/loss:0.004220900358632207 - actor/grad_norm:0.2403564453125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.62080669403076 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:69 - training/epoch:0 - critic/score/mean:0.0341796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0341796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005319633055478334 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.005319633055478334 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:95.5224609375 - response_length/max:128.0 - response_length/min:5.0 - response_length/clip_ratio:0.513671875 - response_length_non_aborted/mean:95.5224609375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:5.0 - response_length_non_aborted/clip_ratio:0.513671875 - response/aborted_ratio:0.0 - prompt_length/mean:62.59375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.141552656998101 - timing_s/old_log_prob:2.3803246390016284 - timing_s/adv:0.19142841499706265 - timing_s/update_actor:4.887370816002658 - timing_s/update_weights:1.0166587620042264 - timing_s/step:12.79537246000109 - timing_per_token_ms/adv:0.001182306421410915 - timing_per_token_ms/update_actor:0.030185539067775866 - timing_per_token_ms/gen:0.042340670214160415 - perf/total_num_tokens:161911 - perf/time_per_step:12.79537246000109 - perf/throughput:12653.871585695615 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:70 - global_seqlen/min:156710.0 - global_seqlen/max:156710.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156710.0 - global_seqlen/balanced_max:156710.0 - global_seqlen/mean:156710.0 - actor/entropy:1.6473016738891602 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.28865885734558105 - training/rollout_probs_diff_mean:0.006728446111083031 - training/rollout_probs_diff_std:0.011118415743112564 - training/rollout_actor_probs_pearson_corr:0.9994672536849976 - rollout_corr/training_ppl:9.483490943908691 - rollout_corr/training_log_ppl:1.7041065692901611 - rollout_corr/kl:0.0012399209663271904 - rollout_corr/k3_kl:0.0016125441761687398 - rollout_corr/rollout_ppl:9.48512077331543 - rollout_corr/rollout_log_ppl:1.7030049562454224 - rollout_corr/log_ppl_diff:0.0011016848729923368 - rollout_corr/log_ppl_abs_diff:0.005943591240793467 - rollout_corr/log_ppl_diff_max:0.0632321834564209 - rollout_corr/log_ppl_diff_min:-0.08223819732666016 - rollout_corr/ppl_ratio:1.0011463165283203 - rollout_corr/chi2_token:0.003974318504333496 - rollout_corr/chi2_seq:0.4198276996612549 - actor/pg_clipfrac:0.0006500043267199848 - actor/ppo_kl:0.00017343277121047623 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0013010711190872826 - actor/loss:-0.0013010705588385463 - actor/grad_norm:0.26751708984375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.590627193450928 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.033) - val-core/char_count/acc/mean@1:np.float64(0.033) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:70 - training/epoch:0 - critic/score/mean:0.03125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.03125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.001069190795533359 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.001069190795533359 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.380859375 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4638671875 - response_length_non_aborted/mean:91.380859375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4638671875 - response/aborted_ratio:0.0 - prompt_length/mean:61.65625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.1330890620010905 - timing_s/old_log_prob:2.3536242759946617 - timing_s/adv:0.1899822880004649 - timing_s/update_actor:4.838258526993741 - timing_s/update_weights:1.0119022639992181 - timing_s/step:12.698797891993308 - timing_s/testing:5.0299104069999885 - timing_per_token_ms/adv:0.001212317580246729 - timing_per_token_ms/update_actor:0.03087396162972204 - timing_per_token_ms/gen:0.044169203646323665 - perf/total_num_tokens:156710 - perf/time_per_step:12.698797891993308 - perf/throughput:12340.538162183593 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:71 - global_seqlen/min:160284.0 - global_seqlen/max:160284.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160284.0 - global_seqlen/balanced_max:160284.0 - global_seqlen/mean:160284.0 - actor/entropy:1.6833723783493042 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.39458245038986206 - training/rollout_probs_diff_mean:0.006773848552256823 - training/rollout_probs_diff_std:0.011327476240694523 - training/rollout_actor_probs_pearson_corr:0.9994500875473022 - rollout_corr/training_ppl:9.185670852661133 - rollout_corr/training_log_ppl:1.7115528583526611 - rollout_corr/kl:0.001955918036401272 - rollout_corr/k3_kl:0.0016328273341059685 - rollout_corr/rollout_ppl:9.168736457824707 - rollout_corr/rollout_log_ppl:1.7092435359954834 - rollout_corr/log_ppl_diff:0.0023093486670404673 - rollout_corr/log_ppl_abs_diff:0.0059129297733306885 - rollout_corr/log_ppl_diff_max:0.051255226135253906 - rollout_corr/log_ppl_diff_min:-0.0768136978149414 - rollout_corr/ppl_ratio:1.0023479461669922 - rollout_corr/chi2_token:0.0026547908782958984 - rollout_corr/chi2_seq:0.32080376148223877 - actor/pg_clipfrac:0.0005285467511081758 - actor/ppo_kl:-0.0002162416617655557 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0024980159651022404 - actor/loss:-0.0024980162270367146 - actor/grad_norm:0.2293701171875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.666009426116943 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:71 - training/epoch:1 - critic/score/mean:0.03125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.03125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.001303993514738977 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.001303993514738977 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.68359375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.513671875 - response_length_non_aborted/mean:94.68359375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.513671875 - response/aborted_ratio:0.0 - prompt_length/mean:61.84375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:4.132510339994042 - timing_s/old_log_prob:2.4033209499975783 - timing_s/adv:0.432868238996889 - timing_s/update_actor:4.8835286920002545 - timing_s/update_weights:1.0249826170038432 - timing_s/step:13.187959138995211 - timing_per_token_ms/adv:0.0027006328703856218 - timing_per_token_ms/update_actor:0.03046797367173426 - timing_per_token_ms/gen:0.04262253331401916 - perf/total_num_tokens:160284 - perf/time_per_step:13.187959138995211 - perf/throughput:12153.81381688236 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 20%|██ | 71/350 [16:34<1:06:43, 14.35s/it] (TaskRunnerV1 pid=2350) Training Progress: 21%|██ | 72/350 [16:47<1:04:55, 14.01s/it] (TaskRunnerV1 pid=2350) Training Progress: 21%|██ | 73/350 [17:00<1:03:21, 13.73s/it] (TaskRunnerV1 pid=2350) step:72 - global_seqlen/min:155140.0 - global_seqlen/max:155140.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155140.0 - global_seqlen/balanced_max:155140.0 - global_seqlen/mean:155140.0 - actor/entropy:1.7087525129318237 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.23117119073867798 - training/rollout_probs_diff_mean:0.006799930706620216 - training/rollout_probs_diff_std:0.011090326122939587 - training/rollout_actor_probs_pearson_corr:0.999463677406311 - rollout_corr/training_ppl:11.296960830688477 - rollout_corr/training_log_ppl:1.7538474798202515 - rollout_corr/kl:0.0014897361397743225 - rollout_corr/k3_kl:0.0016772736562415957 - rollout_corr/rollout_ppl:11.256952285766602 - rollout_corr/rollout_log_ppl:1.7524924278259277 - rollout_corr/log_ppl_diff:0.0013551650336012244 - rollout_corr/log_ppl_abs_diff:0.006464916281402111 - rollout_corr/log_ppl_diff_max:0.07326459884643555 - rollout_corr/log_ppl_diff_min:-0.09609031677246094 - rollout_corr/ppl_ratio:1.0014064311981201 - rollout_corr/chi2_token:0.003743410110473633 - rollout_corr/chi2_seq:0.380435585975647 - actor/pg_clipfrac:0.0006203250735173544 - actor/ppo_kl:0.0003713290713373378 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0043709896999644116 - actor/loss:-0.004370989394374192 - actor/grad_norm:0.27239990234375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.407663345336914 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:72 - training/epoch:1 - critic/score/mean:0.0341796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0341796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0012744261184707284 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.0012744261184707284 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:90.08203125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4755859375 - response_length_non_aborted/mean:90.08203125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4755859375 - response/aborted_ratio:0.0 - prompt_length/mean:61.421875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.115972274994419 - timing_s/old_log_prob:2.32025452900416 - timing_s/adv:0.24105747800058452 - timing_s/update_actor:5.043201291999139 - timing_s/update_weights:1.0177211699992768 - timing_s/step:13.101642471003288 - timing_per_token_ms/adv:0.0015538060977219577 - timing_per_token_ms/update_actor:0.03250742098748962 - timing_per_token_ms/gen:0.04462048778234269 - perf/total_num_tokens:155140 - perf/time_per_step:13.101642471003288 - perf/throughput:11841.263440317327 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:73 - global_seqlen/min:158674.0 - global_seqlen/max:158674.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158674.0 - global_seqlen/balanced_max:158674.0 - global_seqlen/mean:158674.0 - actor/entropy:1.666130542755127 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2814248204231262 - training/rollout_probs_diff_mean:0.006889996584504843 - training/rollout_probs_diff_std:0.01137549988925457 - training/rollout_actor_probs_pearson_corr:0.9994369149208069 - rollout_corr/training_ppl:8.76102352142334 - rollout_corr/training_log_ppl:1.7225667238235474 - rollout_corr/kl:0.001635376363992691 - rollout_corr/k3_kl:0.0016804102342575788 - rollout_corr/rollout_ppl:8.752368927001953 - rollout_corr/rollout_log_ppl:1.7208569049835205 - rollout_corr/log_ppl_diff:0.0017098141834139824 - rollout_corr/log_ppl_abs_diff:0.006512863095849752 - rollout_corr/log_ppl_diff_max:0.09600472450256348 - rollout_corr/log_ppl_diff_min:-0.07824563980102539 - rollout_corr/ppl_ratio:1.0017625093460083 - rollout_corr/chi2_token:0.003461003303527832 - rollout_corr/chi2_seq:0.4723966121673584 - actor/pg_clipfrac:0.0007626965270901566 - actor/ppo_kl:0.0003412082317784856 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.010036182298790663 - actor/loss:-0.010036182939074934 - actor/grad_norm:0.2523193359375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.300047874450684 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:73 - training/epoch:1 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.006130735389888287 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.006130735389888287 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.064453125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4951171875 - response_length_non_aborted/mean:93.064453125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4951171875 - response/aborted_ratio:0.0 - prompt_length/mean:61.890625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.1355002199998125 - timing_s/old_log_prob:2.361632180000015 - timing_s/adv:0.23346632500033593 - timing_s/update_actor:4.8097401409977465 - timing_s/update_weights:1.0237395349977305 - timing_s/step:12.742719042995304 - timing_per_token_ms/adv:0.0014713584141090282 - timing_per_token_ms/update_actor:0.03031208730477423 - timing_per_token_ms/gen:0.043395456567816876 - perf/total_num_tokens:158674 - perf/time_per_step:12.742719042995304 - perf/throughput:12452.130464825981 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:74 - global_seqlen/min:158169.0 - global_seqlen/max:158169.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158169.0 - global_seqlen/balanced_max:158169.0 - global_seqlen/mean:158169.0 - actor/entropy:1.6472636461257935 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2176033854484558 - training/rollout_probs_diff_mean:0.006905312649905682 - training/rollout_probs_diff_std:0.011315809562802315 - training/rollout_actor_probs_pearson_corr:0.9994437098503113 - rollout_corr/training_ppl:10.147588729858398 - rollout_corr/training_log_ppl:1.7336668968200684 - rollout_corr/kl:0.0015816993545740843 - rollout_corr/k3_kl:0.001689854427240789 - rollout_corr/rollout_ppl:10.049769401550293 - rollout_corr/rollout_log_ppl:1.7315388917922974 - rollout_corr/log_ppl_diff:0.0021280355285853148 - rollout_corr/log_ppl_abs_diff:0.006593964993953705 - rollout_corr/log_ppl_diff_max:0.14664602279663086 - rollout_corr/log_ppl_diff_min:-0.04181170463562012 - rollout_corr/ppl_ratio:1.0021919012069702 - rollout_corr/chi2_token:0.0036094188690185547 - rollout_corr/chi2_seq:0.4577983617782593 - actor/pg_clipfrac:0.0010782234451956658 - actor/ppo_kl:3.876292529033282e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.005684736650437117 - actor/loss:0.0056847367086447775 - actor/grad_norm:0.260498046875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.282336711883545 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:74 - training/epoch:1 - critic/score/mean:0.0400390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0400390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.011862101033329964 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.011862101033329964 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.2431640625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.482421875 - response_length_non_aborted/mean:92.2431640625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.482421875 - response/aborted_ratio:0.0 - prompt_length/mean:62.21875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.1015625 - timing_s/gen:4.129727972998808 - timing_s/old_log_prob:2.352888706998783 - timing_s/adv:0.24390240300272126 - timing_s/update_actor:4.862021446002473 - timing_s/update_weights:1.0262178939956357 - timing_s/step:12.789365248994727 - timing_per_token_ms/adv:0.0015420367012671337 - timing_per_token_ms/update_actor:0.030739408139410836 - timing_per_token_ms/gen:0.04372071919496499 - perf/total_num_tokens:158169 - perf/time_per_step:12.789365248994727 - perf/throughput:12367.228312009654 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 21%|██ | 74/350 [17:13<1:02:19, 13.55s/it] (TaskRunnerV1 pid=2350) Training Progress: 21%|██▏ | 75/350 [17:31<1:07:58, 14.83s/it] (TaskRunnerV1 pid=2350) Training Progress: 22%|██▏ | 76/350 [17:44<1:05:12, 14.28s/it] (TaskRunnerV1 pid=2350) step:75 - global_seqlen/min:159582.0 - global_seqlen/max:159582.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159582.0 - global_seqlen/balanced_max:159582.0 - global_seqlen/mean:159582.0 - actor/entropy:1.6564372777938843 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.24426361918449402 - training/rollout_probs_diff_mean:0.006753533147275448 - training/rollout_probs_diff_std:0.011243082582950592 - training/rollout_actor_probs_pearson_corr:0.9994585514068604 - rollout_corr/training_ppl:10.330184936523438 - rollout_corr/training_log_ppl:1.722245693206787 - rollout_corr/kl:0.0015768295852467418 - rollout_corr/k3_kl:0.0016683397116139531 - rollout_corr/rollout_ppl:10.338201522827148 - rollout_corr/rollout_log_ppl:1.7209782600402832 - rollout_corr/log_ppl_diff:0.0012673719320446253 - rollout_corr/log_ppl_abs_diff:0.006173118017613888 - rollout_corr/log_ppl_diff_max:0.053983211517333984 - rollout_corr/log_ppl_diff_min:-0.1189422607421875 - rollout_corr/ppl_ratio:1.0013179779052734 - rollout_corr/chi2_token:0.0035321712493896484 - rollout_corr/chi2_seq:0.3955521583557129 - actor/pg_clipfrac:0.0006632038942674019 - actor/ppo_kl:-0.00013057567832674977 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0019811005768133327 - actor/loss:-0.001981100533157587 - actor/grad_norm:0.23095703125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.255166053771973 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.035) - val-core/char_count/acc/mean@1:np.float64(0.035) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:75 - training/epoch:1 - critic/score/mean:0.0322265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0322265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0012709932634606957 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.0012709932634606957 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.623046875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4990234375 - response_length_non_aborted/mean:93.623046875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4990234375 - response/aborted_ratio:0.0 - prompt_length/mean:62.21875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.151215176003461 - timing_s/old_log_prob:2.3519469599996228 - timing_s/adv:0.1895445389964152 - timing_s/update_actor:4.867692793995957 - timing_s/update_weights:1.0302959699984058 - timing_s/step:12.761100333998911 - timing_s/testing:4.879300659005821 - timing_per_token_ms/adv:0.0011877563822762919 - timing_per_token_ms/update_actor:0.030502768445037392 - timing_per_token_ms/gen:0.043300460790690116 - perf/total_num_tokens:159582 - perf/time_per_step:12.761100333998911 - perf/throughput:12505.347957717391 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:76 - global_seqlen/min:160790.0 - global_seqlen/max:160790.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160790.0 - global_seqlen/balanced_max:160790.0 - global_seqlen/mean:160790.0 - actor/entropy:1.6891722679138184 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.18998199701309204 - training/rollout_probs_diff_mean:0.006786268670111895 - training/rollout_probs_diff_std:0.011094426736235619 - training/rollout_actor_probs_pearson_corr:0.9994642734527588 - rollout_corr/training_ppl:10.090544700622559 - rollout_corr/training_log_ppl:1.7168469429016113 - rollout_corr/kl:0.0016728288028389215 - rollout_corr/k3_kl:0.0016726546455174685 - rollout_corr/rollout_ppl:10.10416030883789 - rollout_corr/rollout_log_ppl:1.715802550315857 - rollout_corr/log_ppl_diff:0.0010444453218951821 - rollout_corr/log_ppl_abs_diff:0.005795814096927643 - rollout_corr/log_ppl_diff_max:0.04007554054260254 - rollout_corr/log_ppl_diff_min:-0.1247711181640625 - rollout_corr/ppl_ratio:1.001086711883545 - rollout_corr/chi2_token:0.0034388303756713867 - rollout_corr/chi2_seq:0.3726522922515869 - actor/pg_clipfrac:0.0008182398093291947 - actor/ppo_kl:0.0001902751303077821 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0025981182197938324 - actor/loss:-0.00259811821160838 - actor/grad_norm:0.258544921875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.645806312561035 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:76 - training/epoch:1 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00047873033327050507 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.00047873033327050507 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.912109375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5146484375 - response_length_non_aborted/mean:94.912109375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5146484375 - response/aborted_ratio:0.0 - prompt_length/mean:62.109375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.143010936000792 - timing_s/old_log_prob:2.354416055000911 - timing_s/adv:0.1940530799984117 - timing_s/update_actor:4.97726513499947 - timing_s/update_weights:1.010912637997535 - timing_s/step:12.85866111300129 - timing_per_token_ms/adv:0.001206872815463721 - timing_per_token_ms/update_actor:0.03095506645313434 - timing_per_token_ms/gen:0.042627954892486795 - perf/total_num_tokens:160790 - perf/time_per_step:12.85866111300129 - perf/throughput:12504.412285772623 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:77 - global_seqlen/min:160436.0 - global_seqlen/max:160436.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160436.0 - global_seqlen/balanced_max:160436.0 - global_seqlen/mean:160436.0 - actor/entropy:1.7064719200134277 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.30269739031791687 - training/rollout_probs_diff_mean:0.00669028889387846 - training/rollout_probs_diff_std:0.011015582829713821 - training/rollout_actor_probs_pearson_corr:0.9994783997535706 - rollout_corr/training_ppl:10.247151374816895 - rollout_corr/training_log_ppl:1.770300269126892 - rollout_corr/kl:0.0015802949201315641 - rollout_corr/k3_kl:0.0016654194332659245 - rollout_corr/rollout_ppl:10.249678611755371 - rollout_corr/rollout_log_ppl:1.7691447734832764 - rollout_corr/log_ppl_diff:0.001155506120994687 - rollout_corr/log_ppl_abs_diff:0.006070590578019619 - rollout_corr/log_ppl_diff_max:0.06969296932220459 - rollout_corr/log_ppl_diff_min:-0.08450770378112793 - rollout_corr/ppl_ratio:1.0011963844299316 - rollout_corr/chi2_token:0.003519415855407715 - rollout_corr/chi2_seq:0.45388126373291016 - actor/pg_clipfrac:0.0007485149180865847 - actor/ppo_kl:0.0002474497634163223 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005811029983306071 - actor/loss:-0.005811030132463202 - actor/grad_norm:0.222412109375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.39574956893921 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:77 - training/epoch:1 - critic/score/mean:0.0224609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0224609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.009461875073611736 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.009461875073611736 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.41015625 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.5126953125 - response_length_non_aborted/mean:93.41015625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.5126953125 - response/aborted_ratio:0.0 - prompt_length/mean:63.265625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.144221752998419 - timing_s/old_log_prob:2.3501654060019064 - timing_s/adv:0.18891503900522366 - timing_s/update_actor:4.883600875000411 - timing_s/update_weights:1.0124961260007694 - timing_s/step:12.764016877998074 - timing_per_token_ms/adv:0.0011775102782743503 - timing_per_token_ms/update_actor:0.03043955767409067 - timing_per_token_ms/gen:0.043326033465044315 - perf/total_num_tokens:160436 - perf/time_per_step:12.764016877998074 - perf/throughput:12569.397356137231 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 22%|██▏ | 77/350 [17:57<1:03:19, 13.92s/it] (TaskRunnerV1 pid=2350) Training Progress: 22%|██▏ | 78/350 [18:10<1:01:52, 13.65s/it] (TaskRunnerV1 pid=2350) Training Progress: 23%|██▎ | 79/350 [18:23<1:00:41, 13.44s/it] (TaskRunnerV1 pid=2350) step:78 - global_seqlen/min:155623.0 - global_seqlen/max:155623.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155623.0 - global_seqlen/balanced_max:155623.0 - global_seqlen/mean:155623.0 - actor/entropy:1.684841513633728 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.1835293471813202 - training/rollout_probs_diff_mean:0.006897905375808477 - training/rollout_probs_diff_std:0.011233244091272354 - training/rollout_actor_probs_pearson_corr:0.9994457960128784 - rollout_corr/training_ppl:10.740220069885254 - rollout_corr/training_log_ppl:1.7651842832565308 - rollout_corr/kl:0.0019548549316823483 - rollout_corr/k3_kl:0.0016373031539842486 - rollout_corr/rollout_ppl:10.747204780578613 - rollout_corr/rollout_log_ppl:1.7631189823150635 - rollout_corr/log_ppl_diff:0.0020652387756854296 - rollout_corr/log_ppl_abs_diff:0.006532709114253521 - rollout_corr/log_ppl_diff_max:0.1010899543762207 - rollout_corr/log_ppl_diff_min:-0.06303882598876953 - rollout_corr/ppl_ratio:1.002115249633789 - rollout_corr/chi2_token:0.0026426315307617188 - rollout_corr/chi2_seq:0.26923704147338867 - actor/pg_clipfrac:0.0009429161291336642 - actor/ppo_kl:-0.00015348102328971436 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.000864048459334299 - actor/loss:-0.0008640482556074858 - actor/grad_norm:0.26318359375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.32065725326538 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:78 - training/epoch:1 - critic/score/mean:0.037109375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.037109375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.009433943778276443 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.009433943778276443 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.1474609375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4814453125 - response_length_non_aborted/mean:91.1474609375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4814453125 - response/aborted_ratio:0.0 - prompt_length/mean:60.828125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.138993288994243 - timing_s/old_log_prob:2.329236466997827 - timing_s/adv:0.19226070400327444 - timing_s/update_actor:4.847789905994432 - timing_s/update_weights:1.0227270539980964 - timing_s/step:12.709992202995636 - timing_per_token_ms/adv:0.0012354260231667198 - timing_per_token_ms/update_actor:0.031150857559579446 - timing_per_token_ms/gen:0.044345564782710054 - perf/total_num_tokens:155623 - perf/time_per_step:12.709992202995636 - perf/throughput:12244.14598486701 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:79 - global_seqlen/min:159322.0 - global_seqlen/max:159322.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159322.0 - global_seqlen/balanced_max:159322.0 - global_seqlen/mean:159322.0 - actor/entropy:1.7020668983459473 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.324638694524765 - training/rollout_probs_diff_mean:0.006830121390521526 - training/rollout_probs_diff_std:0.011390630155801773 - training/rollout_actor_probs_pearson_corr:0.9994407296180725 - rollout_corr/training_ppl:13.726861953735352 - rollout_corr/training_log_ppl:1.7750608921051025 - rollout_corr/kl:0.0017925366992130876 - rollout_corr/k3_kl:0.0016823125770315528 - rollout_corr/rollout_ppl:13.338659286499023 - rollout_corr/rollout_log_ppl:1.7732027769088745 - rollout_corr/log_ppl_diff:0.0018580652540549636 - rollout_corr/log_ppl_abs_diff:0.006240180693566799 - rollout_corr/log_ppl_diff_max:0.15602588653564453 - rollout_corr/log_ppl_diff_min:-0.0600125789642334 - rollout_corr/ppl_ratio:1.0019135475158691 - rollout_corr/chi2_token:0.003162384033203125 - rollout_corr/chi2_seq:0.23303961753845215 - actor/pg_clipfrac:0.0006939689222538922 - actor/ppo_kl:0.0002713633575694985 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0009025703730003443 - actor/loss:0.0009025710169225931 - actor/grad_norm:0.212890625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.305187702178955 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:79 - training/epoch:1 - critic/score/mean:0.025390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.025390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.002257476095110178 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.002257476095110178 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.228515625 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.5029296875 - response_length_non_aborted/mean:93.228515625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.5029296875 - response/aborted_ratio:0.0 - prompt_length/mean:62.359375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:4.139019713999005 - timing_s/old_log_prob:2.357423690998985 - timing_s/adv:0.18305771799350623 - timing_s/update_actor:4.954590894994908 - timing_s/update_weights:1.0094652250045328 - timing_s/step:12.81975611300004 - timing_per_token_ms/adv:0.00114897953825276 - timing_per_token_ms/update_actor:0.03109797074474905 - timing_per_token_ms/gen:0.04335595619381775 - perf/total_num_tokens:159322 - perf/time_per_step:12.81975611300004 - perf/throughput:12427.849531274425 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:80 - global_seqlen/min:163680.0 - global_seqlen/max:163680.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:163680.0 - global_seqlen/balanced_max:163680.0 - global_seqlen/mean:163680.0 - actor/entropy:1.670729160308838 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.27586451172828674 - training/rollout_probs_diff_mean:0.006773154251277447 - training/rollout_probs_diff_std:0.011165295727550983 - training/rollout_actor_probs_pearson_corr:0.9994598031044006 - rollout_corr/training_ppl:10.348834037780762 - rollout_corr/training_log_ppl:1.7489979267120361 - rollout_corr/kl:0.001668460899963975 - rollout_corr/k3_kl:0.001637284061871469 - rollout_corr/rollout_ppl:10.230938911437988 - rollout_corr/rollout_log_ppl:1.7471612691879272 - rollout_corr/log_ppl_diff:0.0018367074662819505 - rollout_corr/log_ppl_abs_diff:0.006053130608052015 - rollout_corr/log_ppl_diff_max:0.16727519035339355 - rollout_corr/log_ppl_diff_min:-0.05937397480010986 - rollout_corr/ppl_ratio:1.00189208984375 - rollout_corr/chi2_token:0.003213167190551758 - rollout_corr/chi2_seq:0.34257781505584717 - actor/pg_clipfrac:0.0007741270662033155 - actor/ppo_kl:-0.00030287504996332853 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.004686844597017625 - actor/loss:-0.004686844884417951 - actor/grad_norm:0.2562255859375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.301852226257324 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.032) - val-core/char_count/acc/mean@1:np.float64(0.032) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:80 - training/epoch:1 - critic/score/mean:0.0341796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0341796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0034506425727158785 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.0034506425727158785 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:96.625 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.53515625 - response_length_non_aborted/mean:96.625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.53515625 - response/aborted_ratio:0.0 - prompt_length/mean:63.21875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.128780645994993 - timing_s/old_log_prob:2.5494189279997954 - timing_s/adv:0.18728963299508905 - timing_s/update_actor:4.947453899003449 - timing_s/update_weights:1.0235225450014696 - timing_s/step:13.013732786996115 - timing_s/testing:5.888398306000454 - timing_per_token_ms/adv:0.001144242625825324 - timing_per_token_ms/update_actor:0.030226380125876397 - timing_per_token_ms/gen:0.04172845898685108 - perf/total_num_tokens:163680 - perf/time_per_step:13.013732786996115 - perf/throughput:12577.482777543744 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 23%|██▎ | 80/350 [18:42<1:08:00, 15.11s/it] (TaskRunnerV1 pid=2350) Training Progress: 23%|██▎ | 81/350 [18:55<1:05:03, 14.51s/it] (TransferQueueController pid=2570) 2026-11-01 17:09:14,619 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_ebda4368: [Performance] Total success requests: 105423, Total req/min: 20752.38, Total avg process time: 0.0001s; (TransferQueueController pid=2570) Time range: last 5.08 minutes; (TransferQueueController pid=2570) Per-operation statistics: KV_RETRIEVE_META: req_count=39679, req/min=7810.76, avg_time=0.000266s, max_time=0.078902s, min_time=0.000062s; NOTIFY_DATA_UPDATE: req_count=26062, req/min=5130.27, avg_time=0.000089s, max_time=0.000989s, min_time=0.000030s; SET_CUSTOM_META: req_count=39533, req/min=7782.02, avg_time=0.000011s, max_time=0.000936s, min_time=0.000003s; KV_LIST: req_count=76, req/min=14.96, avg_time=0.000330s, max_time=0.001131s, min_time=0.000123s; CLEAR_META: req_count=51, req/min=10.04, avg_time=0.001737s, max_time=0.003817s, min_time=0.000868s; KV_RETRIEVE_KEYS: req_count=22, req/min=4.33, avg_time=0.000148s, max_time=0.000258s, min_time=0.000106s (SimpleStorageUnit pid=2676) Per-operation statistics: PUT_DATA: req_count=3306, req/min=650.80, avg_time=0.000595s, max_time=0.001729s, min_time=0.000312s; CLEAR_DATA: req_count=51, req/min=10.04, avg_time=0.001498s, max_time=0.004187s, min_time=0.000377s; GET_DATA: req_count=118, req/min=23.23, avg_time=0.000632s, max_time=0.001247s, min_time=0.000393s (TaskRunnerV1 pid=2350) Training Progress: 23%|██▎ | 82/350 [19:08<1:03:03, 14.12s/it] (TaskRunnerV1 pid=2350) step:81 - global_seqlen/min:158229.0 - global_seqlen/max:158229.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158229.0 - global_seqlen/balanced_max:158229.0 - global_seqlen/mean:158229.0 - actor/entropy:1.6801373958587646 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.288180410861969 - training/rollout_probs_diff_mean:0.00688590481877327 - training/rollout_probs_diff_std:0.011419952847063541 - training/rollout_actor_probs_pearson_corr:0.9994342923164368 - rollout_corr/training_ppl:9.791594505310059 - rollout_corr/training_log_ppl:1.7358050346374512 - rollout_corr/kl:0.001776557182893157 - rollout_corr/k3_kl:0.0016669848700985312 - rollout_corr/rollout_ppl:9.756322860717773 - rollout_corr/rollout_log_ppl:1.7337027788162231 - rollout_corr/log_ppl_diff:0.0021023883018642664 - rollout_corr/log_ppl_abs_diff:0.006183474790304899 - rollout_corr/log_ppl_diff_max:0.05808103084564209 - rollout_corr/log_ppl_diff_min:-0.035712242126464844 - rollout_corr/ppl_ratio:1.0021443367004395 - rollout_corr/chi2_token:0.0031238794326782227 - rollout_corr/chi2_seq:0.36243772506713867 - actor/pg_clipfrac:0.0006941627980268095 - actor/ppo_kl:3.150250267935917e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0015569645183859393 - actor/loss:0.0015569645911455154 - actor/grad_norm:0.230224609375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.355892658233643 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:81 - training/epoch:1 - critic/score/mean:0.0322265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0322265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0016899777110666037 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:-0.0016899777110666037 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:92.3955078125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.49609375 - response_length_non_aborted/mean:92.3955078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.49609375 - response/aborted_ratio:0.0 - prompt_length/mean:62.125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0234375 - timing_s/gen:4.128429003998463 - timing_s/old_log_prob:2.3419889750002767 - timing_s/adv:0.24622126699978253 - timing_s/update_actor:4.862965600004827 - timing_s/update_weights:1.0187146500029485 - timing_s/step:12.78124175099947 - timing_per_token_ms/adv:0.0015561070789790906 - timing_per_token_ms/update_actor:0.030733718850557274 - timing_per_token_ms/gen:0.0436349022227227 - perf/total_num_tokens:158229 - perf/time_per_step:12.78124175099947 - perf/throughput:12379.783051019027 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:82 - global_seqlen/min:158891.0 - global_seqlen/max:158891.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158891.0 - global_seqlen/balanced_max:158891.0 - global_seqlen/mean:158891.0 - actor/entropy:1.6491442918777466 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.287900447845459 - training/rollout_probs_diff_mean:0.006711237132549286 - training/rollout_probs_diff_std:0.011075480841100216 - training/rollout_actor_probs_pearson_corr:0.999470591545105 - rollout_corr/training_ppl:9.664217948913574 - rollout_corr/training_log_ppl:1.7106389999389648 - rollout_corr/kl:0.0016042380593717098 - rollout_corr/k3_kl:0.0016139232320711017 - rollout_corr/rollout_ppl:9.621580123901367 - rollout_corr/rollout_log_ppl:1.708268642425537 - rollout_corr/log_ppl_diff:0.0023703216575086117 - rollout_corr/log_ppl_abs_diff:0.006292656064033508 - rollout_corr/log_ppl_diff_max:0.0870676040649414 - rollout_corr/log_ppl_diff_min:-0.04982709884643555 - rollout_corr/ppl_ratio:1.0024235248565674 - rollout_corr/chi2_token:0.0032459497451782227 - rollout_corr/chi2_seq:0.48292553424835205 - actor/pg_clipfrac:0.0007565423226575754 - actor/ppo_kl:0.00014726314808892008 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.005145148053998128 - actor/loss:0.005145147908478975 - actor/grad_norm:0.2486572265625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.306125164031982 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:82 - training/epoch:1 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008337174542248249 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.008337174542248249 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.2607421875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5009765625 - response_length_non_aborted/mean:92.2607421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5009765625 - response/aborted_ratio:0.0 - prompt_length/mean:62.90625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.130501507999725 - timing_s/old_log_prob:2.3857209460038575 - timing_s/adv:0.19883816899528028 - timing_s/update_actor:4.901154453000345 - timing_s/update_weights:1.025168146006763 - timing_s/step:12.816669441999693 - timing_per_token_ms/adv:0.0012514124084767562 - timing_per_token_ms/update_actor:0.030846016785093838 - timing_per_token_ms/gen:0.04372057695686398 - perf/total_num_tokens:158891 - perf/time_per_step:12.816669441999693 - perf/throughput:12397.214480645089 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:83 - global_seqlen/min:157171.0 - global_seqlen/max:157171.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157171.0 - global_seqlen/balanced_max:157171.0 - global_seqlen/mean:157171.0 - actor/entropy:1.69199800491333 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2739863097667694 - training/rollout_probs_diff_mean:0.0068417321890592575 - training/rollout_probs_diff_std:0.011208071373403072 - training/rollout_actor_probs_pearson_corr:0.9994522333145142 - rollout_corr/training_ppl:10.396964073181152 - rollout_corr/training_log_ppl:1.751250982284546 - rollout_corr/kl:0.00128761469386518 - rollout_corr/k3_kl:0.001669557997956872 - rollout_corr/rollout_ppl:10.345004081726074 - rollout_corr/rollout_log_ppl:1.7494889497756958 - rollout_corr/log_ppl_diff:0.0017619719728827477 - rollout_corr/log_ppl_abs_diff:0.005930755287408829 - rollout_corr/log_ppl_diff_max:0.07713127136230469 - rollout_corr/log_ppl_diff_min:-0.06187081336975098 - rollout_corr/ppl_ratio:1.001802921295166 - rollout_corr/chi2_token:0.004124641418457031 - rollout_corr/chi2_seq:0.4658975601196289 - actor/pg_clipfrac:0.0006631287481771626 - actor/ppo_kl:0.0003055478075566498 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0027898527987417765 - actor/loss:-0.0027898529078811407 - actor/grad_norm:0.2548828125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.303359985351562 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:83 - training/epoch:1 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.003945307340472937 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.003945307340472937 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.2373046875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.5009765625 - response_length_non_aborted/mean:92.2373046875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.5009765625 - response/aborted_ratio:0.0 - prompt_length/mean:61.25 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.127909547001764 - timing_s/old_log_prob:2.4412592469961965 - timing_s/adv:0.22873167900252156 - timing_s/update_actor:4.879225828000926 - timing_s/update_weights:1.0142150219980977 - timing_s/step:12.885478364994924 - timing_per_token_ms/adv:0.0014553045981925518 - timing_per_token_ms/update_actor:0.031044059196677035 - timing_per_token_ms/gen:0.043704243967790324 - perf/total_num_tokens:157171 - perf/time_per_step:12.885478364994924 - perf/throughput:12197.529307641029 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 24%|██▎ | 83/350 [19:21<1:01:21, 13.79s/it] (SimpleStorageUnit pid=2691) 2026-11-01 17:09:14,630 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_52739293: [Performance] Total success requests: 3474, Total req/min: 683.86, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2691) Time range: last 5.08 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2691) Per-operation statistics: PUT_DATA: req_count=3305, req/min=650.59, avg_time=0.000595s, max_time=0.002742s, min_time=0.000310s; CLEAR_DATA: req_count=51, req/min=10.04, avg_time=0.001510s, max_time=0.004422s, min_time=0.000392s; GET_DATA: req_count=118, req/min=23.23, avg_time=0.000614s, max_time=0.001259s, min_time=0.000385s [repeated 7x across cluster] (TaskRunnerV1 pid=2350) Training Progress: 24%|██▍ | 84/350 [19:34<1:00:16, 13.59s/it] (TaskRunnerV1 pid=2350) Training Progress: 24%|██▍ | 85/350 [19:52<1:05:52, 14.91s/it] (TaskRunnerV1 pid=2350) step:84 - global_seqlen/min:158637.0 - global_seqlen/max:158637.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158637.0 - global_seqlen/balanced_max:158637.0 - global_seqlen/mean:158637.0 - actor/entropy:1.6960371732711792 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2409575879573822 - training/rollout_probs_diff_mean:0.006810648832470179 - training/rollout_probs_diff_std:0.011216726154088974 - training/rollout_actor_probs_pearson_corr:0.9994561076164246 - rollout_corr/training_ppl:9.40426254272461 - rollout_corr/training_log_ppl:1.732845425605774 - rollout_corr/kl:0.00167037732899189 - rollout_corr/k3_kl:0.0016789566725492477 - rollout_corr/rollout_ppl:9.36995792388916 - rollout_corr/rollout_log_ppl:1.7304726839065552 - rollout_corr/log_ppl_diff:0.0023726634681224823 - rollout_corr/log_ppl_abs_diff:0.0065264347940683365 - rollout_corr/log_ppl_diff_max:0.13401126861572266 - rollout_corr/log_ppl_diff_min:-0.09378242492675781 - rollout_corr/ppl_ratio:1.0024417638778687 - rollout_corr/chi2_token:0.0033903121948242188 - rollout_corr/chi2_seq:0.2836974859237671 - actor/pg_clipfrac:0.0006923997831715194 - actor/ppo_kl:-0.00014879642003506888 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.013412305997917429 - actor/loss:-0.013412305968813598 - actor/grad_norm:0.2340087890625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.333149433135986 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:84 - training/epoch:1 - critic/score/mean:0.0302734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0302734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.015145360492169857 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.015145360492169857 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.1376953125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.494140625 - response_length_non_aborted/mean:93.1376953125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.494140625 - response/aborted_ratio:0.0 - prompt_length/mean:61.78125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.128057314999751 - timing_s/old_log_prob:2.5665413420065306 - timing_s/adv:0.18640477699955227 - timing_s/update_actor:4.917679780999606 - timing_s/update_weights:1.0272227509994991 - timing_s/step:13.006431539004552 - timing_per_token_ms/adv:0.0011750397259123171 - timing_per_token_ms/update_actor:0.030999576271611328 - timing_per_token_ms/gen:0.043283291025759396 - perf/total_num_tokens:158637 - perf/time_per_step:13.006431539004552 - perf/throughput:12196.811979078875 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:85 - global_seqlen/min:155246.0 - global_seqlen/max:155246.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155246.0 - global_seqlen/balanced_max:155246.0 - global_seqlen/mean:155246.0 - actor/entropy:1.6510294675827026 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2764536738395691 - training/rollout_probs_diff_mean:0.006674508098512888 - training/rollout_probs_diff_std:0.011089542880654335 - training/rollout_actor_probs_pearson_corr:0.9994761943817139 - rollout_corr/training_ppl:9.4963960647583 - rollout_corr/training_log_ppl:1.702805519104004 - rollout_corr/kl:0.0015524612972512841 - rollout_corr/k3_kl:0.001626220066100359 - rollout_corr/rollout_ppl:9.489385604858398 - rollout_corr/rollout_log_ppl:1.701033353805542 - rollout_corr/log_ppl_diff:0.0017722854390740395 - rollout_corr/log_ppl_abs_diff:0.006109375506639481 - rollout_corr/log_ppl_diff_max:0.051995038986206055 - rollout_corr/log_ppl_diff_min:-0.0850834846496582 - rollout_corr/ppl_ratio:1.0018138885498047 - rollout_corr/chi2_token:0.0034241676330566406 - rollout_corr/chi2_seq:0.35639119148254395 - actor/pg_clipfrac:0.0006265687807172071 - actor/ppo_kl:2.7491069583144678e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0006916756356076803 - actor/loss:-0.0006916760466992855 - actor/grad_norm:0.22662353515625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.29714870452881 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.03) - val-core/char_count/acc/mean@1:np.float64(0.03) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:85 - training/epoch:1 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0027795189525932074 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.0027795189525932074 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:89.701171875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.4658203125 - response_length_non_aborted/mean:89.701171875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.4658203125 - response/aborted_ratio:0.0 - prompt_length/mean:61.90625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.144145362995914 - timing_s/old_log_prob:2.3096149359989795 - timing_s/adv:0.40512414799741236 - timing_s/update_actor:5.135675464000087 - timing_s/update_weights:1.0174208030002774 - timing_s/step:13.186937489997945 - timing_s/testing:4.679094947001431 - timing_per_token_ms/adv:0.0026095625523196242 - timing_per_token_ms/update_actor:0.033080887520451976 - timing_per_token_ms/gen:0.04511665646565108 - perf/total_num_tokens:155246 - perf/time_per_step:13.186937489997945 - perf/throughput:11772.710693271376 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:86 - global_seqlen/min:160121.0 - global_seqlen/max:160121.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160121.0 - global_seqlen/balanced_max:160121.0 - global_seqlen/mean:160121.0 - actor/entropy:1.6890966892242432 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2377421259880066 - training/rollout_probs_diff_mean:0.0068615204654634 - training/rollout_probs_diff_std:0.011473107151687145 - training/rollout_actor_probs_pearson_corr:0.9994316101074219 - rollout_corr/training_ppl:9.081158638000488 - rollout_corr/training_log_ppl:1.7267035245895386 - rollout_corr/kl:0.00217080838046968 - rollout_corr/k3_kl:0.0016772357048466802 - rollout_corr/rollout_ppl:9.056259155273438 - rollout_corr/rollout_log_ppl:1.724137544631958 - rollout_corr/log_ppl_diff:0.002566093346104026 - rollout_corr/log_ppl_abs_diff:0.006050222087651491 - rollout_corr/log_ppl_diff_max:0.05493617057800293 - rollout_corr/log_ppl_diff_min:-0.04270124435424805 - rollout_corr/ppl_ratio:1.0026048421859741 - rollout_corr/chi2_token:0.002373337745666504 - rollout_corr/chi2_seq:0.23978638648986816 - actor/pg_clipfrac:0.000507084181663231 - actor/ppo_kl:-0.00020200208382448182 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.001740292616887018 - actor/loss:0.0017402925295755267 - actor/grad_norm:0.16400146484375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.557438850402832 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:86 - training/epoch:1 - critic/score/mean:0.0185546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0185546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00020045378187205642 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.00020045378187205642 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.7275390625 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.5068359375 - response_length_non_aborted/mean:94.7275390625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.5068359375 - response/aborted_ratio:0.0 - prompt_length/mean:61.640625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.138126537000062 - timing_s/old_log_prob:2.3958467609991203 - timing_s/adv:0.1839743499949691 - timing_s/update_actor:4.935640618001344 - timing_s/update_weights:1.0220118969955365 - timing_s/step:13.066556270001456 - timing_per_token_ms/adv:0.0011489707783174541 - timing_per_token_ms/update_actor:0.030824442877582226 - timing_per_token_ms/gen:0.04266065851898498 - perf/total_num_tokens:160121 - perf/time_per_step:13.066556270001456 - perf/throughput:12254.261696144837 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 25%|██▍ | 86/350 [20:05<1:03:20, 14.40s/it] (TaskRunnerV1 pid=2350) Training Progress: 25%|██▍ | 87/350 [20:19<1:01:29, 14.03s/it] (TaskRunnerV1 pid=2350) Training Progress: 25%|██▌ | 88/350 [20:32<1:00:03, 13.75s/it] (TaskRunnerV1 pid=2350) step:87 - global_seqlen/min:160787.0 - global_seqlen/max:160787.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160787.0 - global_seqlen/balanced_max:160787.0 - global_seqlen/mean:160787.0 - actor/entropy:1.663223385810852 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.27082934975624084 - training/rollout_probs_diff_mean:0.006611890159547329 - training/rollout_probs_diff_std:0.010878113098442554 - training/rollout_actor_probs_pearson_corr:0.9994924664497375 - rollout_corr/training_ppl:10.021656036376953 - rollout_corr/training_log_ppl:1.7225925922393799 - rollout_corr/kl:0.0011967378668487072 - rollout_corr/k3_kl:0.0016589928418397903 - rollout_corr/rollout_ppl:9.986065864562988 - rollout_corr/rollout_log_ppl:1.7214912176132202 - rollout_corr/log_ppl_diff:0.0011010176967829466 - rollout_corr/log_ppl_abs_diff:0.0059570735320448875 - rollout_corr/log_ppl_diff_max:0.06844377517700195 - rollout_corr/log_ppl_diff_min:-0.04451560974121094 - rollout_corr/ppl_ratio:1.0011402368545532 - rollout_corr/chi2_token:0.004245877265930176 - rollout_corr/chi2_seq:0.49973320960998535 - actor/pg_clipfrac:0.0007693610526378809 - actor/ppo_kl:7.130582063534047e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.001741887943353504 - actor/loss:0.0017418874194845557 - actor/grad_norm:0.2119140625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.427906036376953 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:87 - training/epoch:1 - critic/score/mean:0.02734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.02734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.003549549961462617 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.003549549961462617 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:94.8466796875 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.5205078125 - response_length_non_aborted/mean:94.8466796875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.5205078125 - response/aborted_ratio:0.0 - prompt_length/mean:62.171875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.134972506995837 - timing_s/old_log_prob:2.589934051997261 - timing_s/adv:0.19850162000511773 - timing_s/update_actor:4.9127193289969 - timing_s/update_weights:1.0299801619985374 - timing_s/step:13.039300061995164 - timing_per_token_ms/adv:0.0012345626201441518 - timing_per_token_ms/update_actor:0.030554207299078283 - timing_per_token_ms/gen:0.04257459620271034 - perf/total_num_tokens:160787 - perf/time_per_step:13.039300061995164 - perf/throughput:12330.953290095367 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:88 - global_seqlen/min:157790.0 - global_seqlen/max:157790.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157790.0 - global_seqlen/balanced_max:157790.0 - global_seqlen/mean:157790.0 - actor/entropy:1.6714880466461182 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.4144287407398224 - training/rollout_probs_diff_mean:0.006781426258385181 - training/rollout_probs_diff_std:0.011346586979925632 - training/rollout_actor_probs_pearson_corr:0.9994487762451172 - rollout_corr/training_ppl:10.776041984558105 - rollout_corr/training_log_ppl:1.728459119796753 - rollout_corr/kl:0.0015246629482135177 - rollout_corr/k3_kl:0.0016488460823893547 - rollout_corr/rollout_ppl:10.714061737060547 - rollout_corr/rollout_log_ppl:1.7264704704284668 - rollout_corr/log_ppl_diff:0.0019887855742126703 - rollout_corr/log_ppl_abs_diff:0.006388076581060886 - rollout_corr/log_ppl_diff_max:0.07708382606506348 - rollout_corr/log_ppl_diff_min:-0.059090614318847656 - rollout_corr/ppl_ratio:1.0020368099212646 - rollout_corr/chi2_token:0.003551006317138672 - rollout_corr/chi2_seq:0.3867952823638916 - actor/pg_clipfrac:0.0007547762876735458 - actor/ppo_kl:-0.00011858610332637909 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.006290508696110919 - actor/loss:0.006290509132668376 - actor/grad_norm:0.22747802734375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.374764919281006 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:88 - training/epoch:1 - critic/score/mean:0.0341796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0341796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004863375332206488 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:-0.004863375332206488 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:91.419921875 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4765625 - response_length_non_aborted/mean:91.419921875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4765625 - response/aborted_ratio:0.0 - prompt_length/mean:62.671875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.133873195001797 - timing_s/old_log_prob:2.362179235002259 - timing_s/adv:0.38199725699814735 - timing_s/update_actor:4.897874877999129 - timing_s/update_weights:1.0247160049984814 - timing_s/step:12.981999730000098 - timing_per_token_ms/adv:0.002420921839141564 - timing_per_token_ms/update_actor:0.031040464402047843 - timing_per_token_ms/gen:0.04415870697760802 - perf/total_num_tokens:157790 - perf/time_per_step:12.981999730000098 - perf/throughput:12154.521898145103 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:89 - global_seqlen/min:160093.0 - global_seqlen/max:160093.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160093.0 - global_seqlen/balanced_max:160093.0 - global_seqlen/mean:160093.0 - actor/entropy:1.6848878860473633 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.22838661074638367 - training/rollout_probs_diff_mean:0.006797430105507374 - training/rollout_probs_diff_std:0.011267752386629581 - training/rollout_actor_probs_pearson_corr:0.9994547963142395 - rollout_corr/training_ppl:10.058967590332031 - rollout_corr/training_log_ppl:1.7412476539611816 - rollout_corr/kl:0.0013107429258525372 - rollout_corr/k3_kl:0.0016561727970838547 - rollout_corr/rollout_ppl:10.019314765930176 - rollout_corr/rollout_log_ppl:1.7398453950881958 - rollout_corr/log_ppl_diff:0.0014022852992638946 - rollout_corr/log_ppl_abs_diff:0.006106196437031031 - rollout_corr/log_ppl_diff_max:0.10078859329223633 - rollout_corr/log_ppl_diff_min:-0.057978153228759766 - rollout_corr/ppl_ratio:1.0014477968215942 - rollout_corr/chi2_token:0.004021406173706055 - rollout_corr/chi2_seq:0.4997999668121338 - actor/pg_clipfrac:0.00029178697635264445 - actor/ppo_kl:0.00021297756681330304 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0037310328043531626 - actor/loss:-0.0037310334155336022 - actor/grad_norm:0.19378662109375 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.357187271118164 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:89 - training/epoch:1 - critic/score/mean:0.017578125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.017578125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.005989649798721075 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.005989649798721075 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.1845703125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.521484375 - response_length_non_aborted/mean:94.1845703125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.521484375 - response/aborted_ratio:0.0 - prompt_length/mean:62.15625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.109167959999468 - timing_s/old_log_prob:2.3637352010046015 - timing_s/adv:0.3841649750029319 - timing_s/update_actor:4.890156018998823 - timing_s/update_weights:1.037563917998341 - timing_s/step:12.959627320000436 - timing_per_token_ms/adv:0.0023996363051659465 - timing_per_token_ms/update_actor:0.030545720418749246 - timing_per_token_ms/gen:0.04260633480221336 - perf/total_num_tokens:160093 - perf/time_per_step:12.959627320000436 - perf/throughput:12353.210169317941 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 25%|██▌ | 89/350 [20:45<58:57, 13.55s/it] (TaskRunnerV1 pid=2350) Training Progress: 26%|██▌ | 90/350 [21:03<1:04:18, 14.84s/it] (TaskRunnerV1 pid=2350) Training Progress: 26%|██▌ | 91/350 [21:16<1:01:56, 14.35s/it] (TaskRunnerV1 pid=2350) step:90 - global_seqlen/min:156647.0 - global_seqlen/max:156647.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156647.0 - global_seqlen/balanced_max:156647.0 - global_seqlen/mean:156647.0 - actor/entropy:1.704782485961914 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.21773797273635864 - training/rollout_probs_diff_mean:0.006734544411301613 - training/rollout_probs_diff_std:0.010950030758976936 - training/rollout_actor_probs_pearson_corr:0.9994757771492004 - rollout_corr/training_ppl:9.507861137390137 - rollout_corr/training_log_ppl:1.7657408714294434 - rollout_corr/kl:0.0013859454775229096 - rollout_corr/k3_kl:0.0016101909568533301 - rollout_corr/rollout_ppl:9.494447708129883 - rollout_corr/rollout_log_ppl:1.7640753984451294 - rollout_corr/log_ppl_diff:0.0016655074432492256 - rollout_corr/log_ppl_abs_diff:0.005730238743126392 - rollout_corr/log_ppl_diff_max:0.05353569984436035 - rollout_corr/log_ppl_diff_min:-0.04931235313415527 - rollout_corr/ppl_ratio:1.0016992092132568 - rollout_corr/chi2_token:0.003676891326904297 - rollout_corr/chi2_seq:0.40001821517944336 - actor/pg_clipfrac:0.0007388914060963241 - actor/ppo_kl:0.0004142350777556203 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0011542212478161673 - actor/loss:-0.0011542211286723614 - actor/grad_norm:0.2470703125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.368962287902832 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.03) - val-core/char_count/acc/mean@1:np.float64(0.03) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:90 - training/epoch:1 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00291112600825727 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.00291112600825727 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:91.7568359375 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4853515625 - response_length_non_aborted/mean:91.7568359375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4853515625 - response/aborted_ratio:0.0 - prompt_length/mean:61.21875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.126841791003244 - timing_s/old_log_prob:2.3654261009942275 - timing_s/adv:0.24851654400117695 - timing_s/update_actor:5.0704990700032795 - timing_s/update_weights:1.014370028002304 - timing_s/step:13.002711100001761 - timing_s/testing:4.709830542000418 - timing_per_token_ms/adv:0.0015864749660138844 - timing_per_token_ms/update_actor:0.03236895101727629 - timing_per_token_ms/gen:0.0439217295948578 - perf/total_num_tokens:156647 - perf/time_per_step:13.002711100001761 - perf/throughput:12047.256821693038 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:91 - global_seqlen/min:158260.0 - global_seqlen/max:158260.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158260.0 - global_seqlen/balanced_max:158260.0 - global_seqlen/mean:158260.0 - actor/entropy:1.6678537130355835 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.26353007555007935 - training/rollout_probs_diff_mean:0.006732845213264227 - training/rollout_probs_diff_std:0.01111852191388607 - training/rollout_actor_probs_pearson_corr:0.9994679093360901 - rollout_corr/training_ppl:9.528943061828613 - rollout_corr/training_log_ppl:1.734236717224121 - rollout_corr/kl:0.0014423190150409937 - rollout_corr/k3_kl:0.0016401471802964807 - rollout_corr/rollout_ppl:9.519587516784668 - rollout_corr/rollout_log_ppl:1.7326788902282715 - rollout_corr/log_ppl_diff:0.001557756680995226 - rollout_corr/log_ppl_abs_diff:0.006040321663022041 - rollout_corr/log_ppl_diff_max:0.06340181827545166 - rollout_corr/log_ppl_diff_min:-0.08907794952392578 - rollout_corr/ppl_ratio:1.0015990734100342 - rollout_corr/chi2_token:0.0036939382553100586 - rollout_corr/chi2_seq:0.49667978286743164 - actor/pg_clipfrac:0.0006778864456767527 - actor/ppo_kl:0.00011035212643643416 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.008055092592258006 - actor/loss:-0.008055092534050345 - actor/grad_norm:0.243621826171875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.62010431289673 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:91 - training/epoch:1 - critic/score/mean:0.0263671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0263671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.012625502422451973 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.012625502422451973 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.23828125 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4970703125 - response_length_non_aborted/mean:93.23828125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4970703125 - response/aborted_ratio:0.0 - prompt_length/mean:61.3125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:4.1309672860006685 - timing_s/old_log_prob:2.6310470040043583 - timing_s/adv:0.24547773999802303 - timing_s/update_actor:4.874798258999363 - timing_s/update_weights:1.0206685529992683 - timing_s/step:13.077985142997932 - timing_per_token_ms/adv:0.0015511041324277961 - timing_per_token_ms/update_actor:0.03080246593579782 - timing_per_token_ms/gen:0.04326707534878575 - perf/total_num_tokens:158260 - perf/time_per_step:13.077985142997932 - perf/throughput:12101.252468904493 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:92 - global_seqlen/min:163041.0 - global_seqlen/max:163041.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:163041.0 - global_seqlen/balanced_max:163041.0 - global_seqlen/mean:163041.0 - actor/entropy:1.6943119764328003 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3172437250614166 - training/rollout_probs_diff_mean:0.006763936020433903 - training/rollout_probs_diff_std:0.011201432906091213 - training/rollout_actor_probs_pearson_corr:0.999459981918335 - rollout_corr/training_ppl:10.323324203491211 - rollout_corr/training_log_ppl:1.746788501739502 - rollout_corr/kl:0.0014862552052363753 - rollout_corr/k3_kl:0.0016571572050452232 - rollout_corr/rollout_ppl:10.235969543457031 - rollout_corr/rollout_log_ppl:1.7449331283569336 - rollout_corr/log_ppl_diff:0.0018552105175331235 - rollout_corr/log_ppl_abs_diff:0.0056915730237960815 - rollout_corr/log_ppl_diff_max:0.11027908325195312 - rollout_corr/log_ppl_diff_min:-0.0657505989074707 - rollout_corr/ppl_ratio:1.0018959045410156 - rollout_corr/chi2_token:0.003676772117614746 - rollout_corr/chi2_seq:0.3672577142715454 - actor/pg_clipfrac:0.0005064591581079488 - actor/ppo_kl:0.00015060159252774003 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0017154509805550333 - actor/loss:0.001715451362542808 - actor/grad_norm:0.17822265625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.60585594177246 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:92 - training/epoch:1 - critic/score/mean:0.0185546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0185546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0011032602051272988 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.0011032602051272988 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:95.6103515625 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.5146484375 - response_length_non_aborted/mean:95.6103515625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.5146484375 - response/aborted_ratio:0.0 - prompt_length/mean:63.609375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.138594992997241 - timing_s/old_log_prob:2.402029376004066 - timing_s/adv:0.44857925499673 - timing_s/update_actor:4.921236148002208 - timing_s/update_weights:1.0163170659943717 - timing_s/step:13.12657933599985 - timing_per_token_ms/adv:0.0027513279174976235 - timing_per_token_ms/update_actor:0.030184040505162556 - timing_per_token_ms/gen:0.04227153866500425 - perf/total_num_tokens:163041 - perf/time_per_step:13.12657933599985 - perf/throughput:12420.676844031826 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 26%|██▋ | 92/350 [21:29<1:00:20, 14.03s/it] (TaskRunnerV1 pid=2350) Training Progress: 27%|██▋ | 93/350 [21:42<59:04, 13.79s/it] (TaskRunnerV1 pid=2350) Training Progress: 27%|██▋ | 94/350 [21:56<57:58, 13.59s/it] (TaskRunnerV1 pid=2350) step:93 - global_seqlen/min:160757.0 - global_seqlen/max:160757.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160757.0 - global_seqlen/balanced_max:160757.0 - global_seqlen/mean:160757.0 - actor/entropy:1.6646968126296997 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.2608383297920227 - training/rollout_probs_diff_mean:0.006775864865630865 - training/rollout_probs_diff_std:0.011292430572211742 - training/rollout_actor_probs_pearson_corr:0.9994527697563171 - rollout_corr/training_ppl:9.854936599731445 - rollout_corr/training_log_ppl:1.7044956684112549 - rollout_corr/kl:0.0014762955252081156 - rollout_corr/k3_kl:0.001639185706153512 - rollout_corr/rollout_ppl:9.882976531982422 - rollout_corr/rollout_log_ppl:1.7032279968261719 - rollout_corr/log_ppl_diff:0.001267704414203763 - rollout_corr/log_ppl_abs_diff:0.0061445049941539764 - rollout_corr/log_ppl_diff_max:0.061397552490234375 - rollout_corr/log_ppl_diff_min:-0.07016682624816895 - rollout_corr/ppl_ratio:1.0013140439987183 - rollout_corr/chi2_token:0.003628969192504883 - rollout_corr/chi2_seq:0.44086575508117676 - actor/pg_clipfrac:0.0007253075339879919 - actor/ppo_kl:0.00019355010423775335 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005083108902908862 - actor/loss:-0.005083108437247574 - actor/grad_norm:0.23046875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.448968410491943 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:93 - training/epoch:1 - critic/score/mean:0.02734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.02734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.010615488514304161 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.010615488514304161 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.9892578125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.513671875 - response_length_non_aborted/mean:93.9892578125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.513671875 - response/aborted_ratio:0.0 - prompt_length/mean:63.0 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.1228158450030605 - timing_s/old_log_prob:2.393136584003514 - timing_s/adv:0.38529157800076064 - timing_s/update_actor:4.981261092994828 - timing_s/update_weights:1.0448309040002641 - timing_s/step:13.105798018004862 - timing_per_token_ms/adv:0.00239673282034848 - timing_per_token_ms/update_actor:0.03098627800341402 - timing_per_token_ms/gen:0.042836675619544504 - perf/total_num_tokens:160757 - perf/time_per_step:13.105798018004862 - perf/throughput:12266.097782000807 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:94 - global_seqlen/min:159735.0 - global_seqlen/max:159735.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159735.0 - global_seqlen/balanced_max:159735.0 - global_seqlen/mean:159735.0 - actor/entropy:1.7241474390029907 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.21483755111694336 - training/rollout_probs_diff_mean:0.0068529932759702206 - training/rollout_probs_diff_std:0.011293979361653328 - training/rollout_actor_probs_pearson_corr:0.9994450807571411 - rollout_corr/training_ppl:10.029297828674316 - rollout_corr/training_log_ppl:1.7754712104797363 - rollout_corr/kl:0.0018732179887592793 - rollout_corr/k3_kl:0.001665897318162024 - rollout_corr/rollout_ppl:10.017945289611816 - rollout_corr/rollout_log_ppl:1.7738292217254639 - rollout_corr/log_ppl_diff:0.0016419822350144386 - rollout_corr/log_ppl_abs_diff:0.006167142186313868 - rollout_corr/log_ppl_diff_max:0.085418701171875 - rollout_corr/log_ppl_diff_min:-0.08777570724487305 - rollout_corr/ppl_ratio:1.001686930656433 - rollout_corr/chi2_token:0.0029163360595703125 - rollout_corr/chi2_seq:0.38442742824554443 - actor/pg_clipfrac:0.00036979846421066515 - actor/ppo_kl:-0.00012447878270904766 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0011393974855309352 - actor/loss:-0.0011393975000828505 - actor/grad_norm:0.1851806640625 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.442280769348145 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:94 - training/epoch:1 - critic/score/mean:0.0166015625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0166015625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.000600010622292757 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.000600010622292757 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.3662109375 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.521484375 - response_length_non_aborted/mean:94.3662109375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.521484375 - response/aborted_ratio:0.0 - prompt_length/mean:61.625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.131372727002599 - timing_s/old_log_prob:2.374246688996209 - timing_s/adv:0.18741519800096285 - timing_s/update_actor:5.078968778994749 - timing_s/update_weights:1.0176889059948735 - timing_s/step:12.961330671001633 - timing_per_token_ms/adv:0.0011732882461637265 - timing_per_token_ms/update_actor:0.031796217353709265 - timing_per_token_ms/gen:0.04275411334874522 - perf/total_num_tokens:159735 - perf/time_per_step:12.961330671001633 - perf/throughput:12323.96611540626 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:95 - global_seqlen/min:159452.0 - global_seqlen/max:159452.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159452.0 - global_seqlen/balanced_max:159452.0 - global_seqlen/mean:159452.0 - actor/entropy:1.7021845579147339 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.23045015335083008 - training/rollout_probs_diff_mean:0.006733743008226156 - training/rollout_probs_diff_std:0.011008241213858128 - training/rollout_actor_probs_pearson_corr:0.9994732141494751 - rollout_corr/training_ppl:10.943580627441406 - rollout_corr/training_log_ppl:1.7524354457855225 - rollout_corr/kl:0.0018079029396176338 - rollout_corr/k3_kl:0.00164977484382689 - rollout_corr/rollout_ppl:10.941872596740723 - rollout_corr/rollout_log_ppl:1.750735878944397 - rollout_corr/log_ppl_diff:0.0016998335486277938 - rollout_corr/log_ppl_abs_diff:0.005878083873540163 - rollout_corr/log_ppl_diff_max:0.044336795806884766 - rollout_corr/log_ppl_diff_min:-0.08468294143676758 - rollout_corr/ppl_ratio:1.0017379522323608 - rollout_corr/chi2_token:0.00299227237701416 - rollout_corr/chi2_seq:0.26037800312042236 - actor/pg_clipfrac:0.0007249233543310158 - actor/ppo_kl:-0.0001945391962484185 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.005416099680587649 - actor/loss:-0.005416099913418293 - actor/grad_norm:0.22381591796875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.404078483581543 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.042) - val-core/char_count/acc/mean@1:np.float64(0.042) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:95 - training/epoch:1 - critic/score/mean:0.029296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.029296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.009822781197726727 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.935412585735321 - critic/returns/mean:0.009822781197726727 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.935412585735321 - response_length/mean:93.60546875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.49609375 - response_length_non_aborted/mean:93.60546875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.49609375 - response/aborted_ratio:0.0 - prompt_length/mean:62.109375 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.138781084999209 - timing_s/old_log_prob:2.3680935369993676 - timing_s/adv:0.18178716999682365 - timing_s/update_actor:4.91890023000451 - timing_s/update_weights:1.0237728329957463 - timing_s/step:12.80111356999987 - timing_s/testing:4.834905957002775 - timing_per_token_ms/adv:0.00114007456787512 - timing_per_token_ms/update_actor:0.030848783521087917 - timing_per_token_ms/gen:0.04317887039393241 - perf/total_num_tokens:159452 - perf/time_per_step:12.80111356999987 - perf/throughput:12456.10384815933 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 27%|██▋ | 95/350 [22:13<1:03:04, 14.84s/it] (TaskRunnerV1 pid=2350) Training Progress: 27%|██▋ | 96/350 [22:26<1:00:39, 14.33s/it] (TaskRunnerV1 pid=2350) Training Progress: 28%|██▊ | 97/350 [22:40<59:00, 14.00s/it] (TaskRunnerV1 pid=2350) step:96 - global_seqlen/min:160522.0 - global_seqlen/max:160522.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160522.0 - global_seqlen/balanced_max:160522.0 - global_seqlen/mean:160522.0 - actor/entropy:1.6712732315063477 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.27392491698265076 - training/rollout_probs_diff_mean:0.006677674595266581 - training/rollout_probs_diff_std:0.011251685209572315 - training/rollout_actor_probs_pearson_corr:0.9994649291038513 - rollout_corr/training_ppl:9.711685180664062 - rollout_corr/training_log_ppl:1.7285432815551758 - rollout_corr/kl:0.0016249327454715967 - rollout_corr/k3_kl:0.0016295475652441382 - rollout_corr/rollout_ppl:9.692792892456055 - rollout_corr/rollout_log_ppl:1.7269244194030762 - rollout_corr/log_ppl_diff:0.0016188081353902817 - rollout_corr/log_ppl_abs_diff:0.00613831914961338 - rollout_corr/log_ppl_diff_max:0.08751606941223145 - rollout_corr/log_ppl_diff_min:-0.07437455654144287 - rollout_corr/ppl_ratio:1.001664400100708 - rollout_corr/chi2_token:0.0032743215560913086 - rollout_corr/chi2_seq:0.39327144622802734 - actor/pg_clipfrac:0.00042899686377495527 - actor/ppo_kl:0.0001567278009133588 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.010776601149700582 - actor/loss:-0.010776600800454617 - actor/grad_norm:0.20965576171875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.534794330596924 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:96 - training/epoch:1 - critic/score/mean:0.025390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.025390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.015013583935797215 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.7245674133300781 - critic/returns/mean:0.015013583935797215 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.7245674133300781 - response_length/mean:93.962890625 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.51171875 - response_length_non_aborted/mean:93.962890625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.51171875 - response/aborted_ratio:0.0 - prompt_length/mean:62.796875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:4.141020230999857 - timing_s/old_log_prob:2.342180122002901 - timing_s/adv:0.18421107200265396 - timing_s/update_actor:5.094323115001316 - timing_s/update_weights:1.0463209559966344 - timing_s/step:13.001794941003027 - timing_per_token_ms/adv:0.0011475752358097579 - timing_per_token_ms/update_actor:0.03173598083129612 - timing_per_token_ms/gen:0.04303789551850856 - perf/total_num_tokens:160522 - perf/time_per_step:13.001794941003027 - perf/throughput:12346.141492646591 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:97 - global_seqlen/min:159490.0 - global_seqlen/max:159490.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159490.0 - global_seqlen/balanced_max:159490.0 - global_seqlen/mean:159490.0 - actor/entropy:1.7200779914855957 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.26797202229499817 - training/rollout_probs_diff_mean:0.0068271043710410595 - training/rollout_probs_diff_std:0.011104014702141285 - training/rollout_actor_probs_pearson_corr:0.999458909034729 - rollout_corr/training_ppl:12.263303756713867 - rollout_corr/training_log_ppl:1.7719775438308716 - rollout_corr/kl:0.0014240897726267576 - rollout_corr/k3_kl:0.0016536698676645756 - rollout_corr/rollout_ppl:12.178292274475098 - rollout_corr/rollout_log_ppl:1.7704341411590576 - rollout_corr/log_ppl_diff:0.0015433041844516993 - rollout_corr/log_ppl_abs_diff:0.006124584935605526 - rollout_corr/log_ppl_diff_max:0.05510997772216797 - rollout_corr/log_ppl_diff_min:-0.06939911842346191 - rollout_corr/ppl_ratio:1.0015869140625 - rollout_corr/chi2_token:0.003789067268371582 - rollout_corr/chi2_seq:0.4246504306793213 - actor/pg_clipfrac:0.0006276998362106193 - actor/ppo_kl:9.819690876611273e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0018762487306958064 - actor/loss:0.001876248512417078 - actor/grad_norm:0.2283935546875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.45987892150879 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:97 - training/epoch:1 - critic/score/mean:0.029296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.029296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0018647946417331696 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:-0.0018647946417331696 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:94.861328125 - response_length/max:128.0 - response_length/min:4.0 - response_length/clip_ratio:0.50390625 - response_length_non_aborted/mean:94.861328125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:4.0 - response_length_non_aborted/clip_ratio:0.50390625 - response/aborted_ratio:0.0 - prompt_length/mean:60.890625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:4.143860516000132 - timing_s/old_log_prob:2.386675974004902 - timing_s/adv:0.19347662099607987 - timing_s/update_actor:4.949261817993829 - timing_s/update_weights:1.0381310209995718 - timing_s/step:12.886153471998114 - timing_per_token_ms/adv:0.0012130956235254866 - timing_per_token_ms/update_actor:0.03103180022568079 - timing_per_token_ms/gen:0.042659520640739274 - perf/total_num_tokens:159490 - perf/time_per_step:12.886153471998114 - perf/throughput:12376.850884678284 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:98 - global_seqlen/min:160333.0 - global_seqlen/max:160333.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160333.0 - global_seqlen/balanced_max:160333.0 - global_seqlen/mean:160333.0 - actor/entropy:1.692503571510315 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.18441542983055115 - training/rollout_probs_diff_mean:0.006748953368514776 - training/rollout_probs_diff_std:0.011141295544803143 - training/rollout_actor_probs_pearson_corr:0.9994670152664185 - rollout_corr/training_ppl:11.167734146118164 - rollout_corr/training_log_ppl:1.7613552808761597 - rollout_corr/kl:0.0017421991797164083 - rollout_corr/k3_kl:0.001683558919467032 - rollout_corr/rollout_ppl:11.10427474975586 - rollout_corr/rollout_log_ppl:1.7589085102081299 - rollout_corr/log_ppl_diff:0.0024468472693115473 - rollout_corr/log_ppl_abs_diff:0.006674064788967371 - rollout_corr/log_ppl_diff_max:0.15987157821655273 - rollout_corr/log_ppl_diff_min:-0.05555319786071777 - rollout_corr/ppl_ratio:1.0025120973587036 - rollout_corr/chi2_token:0.0032624006271362305 - rollout_corr/chi2_seq:0.35143041610717773 - actor/pg_clipfrac:0.0009076919001624143 - actor/ppo_kl:9.498000660338838e-06 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.001927954304846935 - actor/loss:-0.0019279543776065111 - actor/grad_norm:0.21923828125 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.399818897247314 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:98 - training/epoch:1 - critic/score/mean:0.0244140625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0244140625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0056282756850123405 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.0056282756850123405 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.5283203125 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.509765625 - response_length_non_aborted/mean:93.5283203125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.509765625 - response/aborted_ratio:0.0 - prompt_length/mean:63.046875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:4.127174897999794 - timing_s/old_log_prob:2.3907664039943484 - timing_s/adv:0.19797188400116283 - timing_s/update_actor:4.902913078003621 - timing_s/update_weights:1.0316746349999448 - timing_s/step:12.823531123998691 - timing_per_token_ms/adv:0.0012347544423241806 - timing_per_token_ms/update_actor:0.03057956302198313 - timing_per_token_ms/gen:0.04309330289329763 - perf/total_num_tokens:160333 - perf/time_per_step:12.823531123998691 - perf/throughput:12503.030440651688 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 28%|██▊ | 98/350 [22:53<57:27, 13.68s/it] (TaskRunnerV1 pid=2350) Training Progress: 28%|██▊ | 99/350 [23:06<56:23, 13.48s/it] (TaskRunnerV1 pid=2350) Training Progress: 29%|██▊ | 100/350 [23:24<1:01:46, 14.82s/it] (TaskRunnerV1 pid=2350) step:99 - global_seqlen/min:159543.0 - global_seqlen/max:159543.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159543.0 - global_seqlen/balanced_max:159543.0 - global_seqlen/mean:159543.0 - actor/entropy:1.6215529441833496 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.22709336876869202 - training/rollout_probs_diff_mean:0.006849639117717743 - training/rollout_probs_diff_std:0.011213825084269047 - training/rollout_actor_probs_pearson_corr:0.9994509816169739 - rollout_corr/training_ppl:10.45439338684082 - rollout_corr/training_log_ppl:1.7113498449325562 - rollout_corr/kl:0.0018917924026027322 - rollout_corr/k3_kl:0.001665956573560834 - rollout_corr/rollout_ppl:10.438322067260742 - rollout_corr/rollout_log_ppl:1.709477186203003 - rollout_corr/log_ppl_diff:0.0018726801499724388 - rollout_corr/log_ppl_abs_diff:0.005965969990938902 - rollout_corr/log_ppl_diff_max:0.05926179885864258 - rollout_corr/log_ppl_diff_min:-0.07845926284790039 - rollout_corr/ppl_ratio:1.0019135475158691 - rollout_corr/chi2_token:0.0029791593551635742 - rollout_corr/chi2_seq:0.29898154735565186 - actor/pg_clipfrac:0.0009120218728639884 - actor/ppo_kl:-2.480336297594475e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.006169575062813237 - actor/loss:-0.006169574800878763 - actor/grad_norm:0.27313232421875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.392114639282227 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:99 - training/epoch:1 - critic/score/mean:0.033203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.033203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.009649617597460747 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.009649617597460747 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:92.6630859375 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.478515625 - response_length_non_aborted/mean:92.6630859375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.478515625 - response/aborted_ratio:0.0 - prompt_length/mean:63.140625 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:4.125857398001244 - timing_s/old_log_prob:2.4477411690022564 - timing_s/adv:0.18873764899763046 - timing_s/update_actor:4.909580423998705 - timing_s/update_weights:1.0295313620008528 - timing_s/step:12.877814436003973 - timing_per_token_ms/adv:0.001182989219192509 - timing_per_token_ms/update_actor:0.030772772381105436 - timing_per_token_ms/gen:0.04348179832855127 - perf/total_num_tokens:159543 - perf/time_per_step:12.877814436003973 - perf/throughput:12388.981126638031 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:100 - global_seqlen/min:158243.0 - global_seqlen/max:158243.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158243.0 - global_seqlen/balanced_max:158243.0 - global_seqlen/mean:158243.0 - actor/entropy:1.7253822088241577 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.3896201252937317 - training/rollout_probs_diff_mean:0.006856846623122692 - training/rollout_probs_diff_std:0.01131791528314352 - training/rollout_actor_probs_pearson_corr:0.9994430541992188 - rollout_corr/training_ppl:12.253820419311523 - rollout_corr/training_log_ppl:1.7843122482299805 - rollout_corr/kl:0.001435249112546444 - rollout_corr/k3_kl:0.00168617011513561 - rollout_corr/rollout_ppl:12.18645191192627 - rollout_corr/rollout_log_ppl:1.783027172088623 - rollout_corr/log_ppl_diff:0.001284917932935059 - rollout_corr/log_ppl_abs_diff:0.00607153819873929 - rollout_corr/log_ppl_diff_max:0.06772327423095703 - rollout_corr/log_ppl_diff_min:-0.0967550277709961 - rollout_corr/ppl_ratio:1.0013295412063599 - rollout_corr/chi2_token:0.0038869380950927734 - rollout_corr/chi2_seq:0.5069923400878906 - actor/pg_clipfrac:0.0007151562288830367 - actor/ppo_kl:-5.2679359517545266e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.010728509058026248 - actor/loss:-0.010728508583270013 - actor/grad_norm:0.25 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.421592235565186 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.035) - val-core/char_count/acc/mean@1:np.float64(0.035) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:100 - training/epoch:1 - critic/score/mean:0.0283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.004724449943751097 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.004724449943751097 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.2216796875 - response_length/max:128.0 - response_length/min:2.0 - response_length/clip_ratio:0.49609375 - response_length_non_aborted/mean:93.2216796875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:2.0 - response_length_non_aborted/clip_ratio:0.49609375 - response/aborted_ratio:0.0 - prompt_length/mean:61.3125 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:4.124755286000436 - timing_s/old_log_prob:2.590570229003788 - timing_s/adv:0.18274779599596513 - timing_s/update_actor:4.887348993004707 - timing_s/update_weights:1.0503899209943484 - timing_s/step:13.02502525899763 - timing_s/testing:4.803299093997339 - timing_per_token_ms/adv:0.0011548554817335688 - timing_per_token_ms/update_actor:0.03088508807975523 - timing_per_token_ms/gen:0.04320970559088652 - perf/total_num_tokens:158243 - perf/time_per_step:13.02502525899763 - perf/throughput:12149.15110361774 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2350) step:101 - global_seqlen/min:159823.0 - global_seqlen/max:159823.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159823.0 - global_seqlen/balanced_max:159823.0 - global_seqlen/mean:159823.0 - actor/entropy:1.7121936082839966 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.24161958694458008 - training/rollout_probs_diff_mean:0.00697504123672843 - training/rollout_probs_diff_std:0.011330231092870235 - training/rollout_actor_probs_pearson_corr:0.9994370937347412 - rollout_corr/training_ppl:10.705047607421875 - rollout_corr/training_log_ppl:1.7694350481033325 - rollout_corr/kl:0.001929734949953854 - rollout_corr/k3_kl:0.0017115069786086679 - rollout_corr/rollout_ppl:10.643993377685547 - rollout_corr/rollout_log_ppl:1.7671924829483032 - rollout_corr/log_ppl_diff:0.002242512535303831 - rollout_corr/log_ppl_abs_diff:0.006524702068418264 - rollout_corr/log_ppl_diff_max:0.08398675918579102 - rollout_corr/log_ppl_diff_min:-0.07639789581298828 - rollout_corr/ppl_ratio:1.0022964477539062 - rollout_corr/chi2_token:0.0029916763305664062 - rollout_corr/chi2_seq:0.3889509439468384 - actor/pg_clipfrac:0.0008037176066864049 - actor/ppo_kl:-0.00021959170481977708 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.00039352106978185475 - actor/loss:-0.00039352127350866795 - actor/grad_norm:0.21746826171875 - actor/perf/max_memory_allocated_gb:4.069221496582031 - actor/perf/max_memory_reserved_gb:4.607421875 - actor/perf/cpu_memory_used_gb:29.567794799804688 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:101 - training/epoch:1 - critic/score/mean:0.0263671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.0263671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00038258457789197564 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-0.5400605201721191 - critic/returns/mean:0.00038258457789197564 - critic/returns/max:2.4748666286468506 - critic/returns/min:-0.5400605201721191 - response_length/mean:93.6552734375 - response_length/max:128.0 - response_length/min:3.0 - response_length/clip_ratio:0.4931640625 - response_length_non_aborted/mean:93.6552734375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:3.0 - response_length_non_aborted/clip_ratio:0.4931640625 - response/aborted_ratio:0.0 - prompt_length/mean:62.421875 - prompt_length/max:77.0 - prompt_length/min:47.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:4.144644195002911 - timing_s/old_log_prob:2.3636386590005714 - timing_s/adv:0.22740653400251176 - timing_s/update_actor:4.930914929995197 - timing_s/update_weights:1.0421937719947891 - timing_s/step:12.883175873001164 - timing_per_token_ms/adv:0.00142286488179118 - timing_per_token_ms/update_actor:0.03085234872324507 - timing_per_token_ms/gen:0.04321704425307771 - perf/total_num_tokens:159823 - perf/time_per_step:12.883175873001164 - perf/throughput:12405.559124201327 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2350) Training Progress: 29%|██▉ | 101/350 [23:37<59:29, 14.34s/it] (TaskRunnerV1 pid=2350) Training Progress: 29%|██▉ | 102/350 [23:50<57:35, 13.94s/it] + REPO=/workspace/verl + cd /workspace/verl + export MLFLOW_TRACKING_URI=http://:5000/mlflow + MLFLOW_TRACKING_URI=http://:5000/mlflow + export MODEL_PATH=/models/Qwen3-1.7B + MODEL_PATH=/models/Qwen3-1.7B + export TRAIN_FILE=/data/char_count/rl/train.parquet + TRAIN_FILE=/data/char_count/rl/train.parquet + export TEST_FILE=/data/char_count/rl/test.parquet + TEST_FILE=/data/char_count/rl/test.parquet + export NGPUS_PER_NODE=1 + NGPUS_PER_NODE=1 + export NNODES=1 + NNODES=1 + export ROLLOUT_TP=1 + ROLLOUT_TP=1 + export TRAIN_BATCH_SIZE=128 + TRAIN_BATCH_SIZE=128 + export PPO_MINI_BATCH_SIZE=16 + PPO_MINI_BATCH_SIZE=16 + export PPO_MICRO_BATCH_SIZE_PER_GPU=8 + PPO_MICRO_BATCH_SIZE_PER_GPU=8 + export LOG_PROB_MICRO_BATCH_SIZE_PER_GPU=8 + LOG_PROB_MICRO_BATCH_SIZE_PER_GPU=8 + export MAX_PROMPT_LENGTH=128 + MAX_PROMPT_LENGTH=128 + export MAX_RESPONSE_LENGTH=128 + MAX_RESPONSE_LENGTH=128 + export ACTOR_LR=1e-6 + ACTOR_LR=1e-6 + export ENTROPY_COEFF=0 + ENTROPY_COEFF=0 + export ROLLOUT_N=8 + ROLLOUT_N=8 + export ROLLOUT_GPU_MEM_UTIL=0.7 + ROLLOUT_GPU_MEM_UTIL=0.7 + export PROJECT_NAME=verl_char_count + PROJECT_NAME=verl_char_count + export EXPERIMENT_NAME=qwen3-1.7b-char-count-grpo-direct-1gpu + EXPERIMENT_NAME=qwen3-1.7b-char-count-grpo-direct-1gpu + export SAVE_FREQ=-1 + SAVE_FREQ=-1 + export TEST_FREQ=5 + TEST_FREQ=5 + export TOTAL_EPOCHS=2 + TOTAL_EPOCHS=2 + uv sync --frozen --all-packages --extra vllm --extra fsdp Uninstalled 33 packages in 91ms Installed 1 package in 2ms - alembic==1.20.1 - blinker==1.9.0 - certifi==2026.10.23 + certifi==2026.5.20 - contourpy==1.4.0 - cycler==0.12.1 - databricks-sdk==0.158.0 - docker==7.2.0 - flask==3.1.3 - flask-cors==6.0.5 - fonttools==4.68.0 - graphene==3.4.3 - graphql-core==3.2.13 - graphql-relay==3.2.0 - gunicorn==26.2.0 - huey==3.5.3 - itsdangerous==2.2.0 - joblib==1.6.0 - kiwisolver==1.5.1 - mako==1.4.3 - matplotlib==3.11.2 - mlflow==3.17.2 - mlflow-skinny==3.17.2 - mlflow-tracing==3.17.2 - narwhals==2.28.0 - prettytable==3.18.0 - pyparsing==3.3.3 - scikit-learn==1.9.1 - scipy==1.18.1 - skops==0.16.0 - sqlalchemy==2.1.6 - sqlparse==0.6.0 - threadpoolctl==3.7.0 - wcwidth==0.9.4 + '[' '!' -f /models/Qwen3-1.7B/config.json ']' + '[' '!' -f /data/char_count/rl/train.parquet ']' + '[' '!' -f /data/char_count/rl/test.parquet ']' + UV_OFFLINE=0 + uv pip install --python .venv/bin/python -q mlflow==3.17.2 + .venv/bin/python -c 'import mlflow, pandas; print(f'\''env ok: mlflow {mlflow.__version__} | pandas {pandas.__version__}'\'')' env ok: mlflow 3.17.2 | pandas 3.0.3 + export UV_NO_SYNC=1 + UV_NO_SYNC=1 + exec bash ./examples/grpo_trainer/run_qwen3_4b_fsdp.sh actor_rollout_ref.actor.strategy=fsdp2 actor_rollout_ref.actor.fsdp_config.param_offload=True actor_rollout_ref.actor.fsdp_config.optimizer_offload=True actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 '++data.apply_chat_template_kwargs={enable_thinking: false}' actor_rollout_ref.actor.use_kl_loss=False actor_rollout_ref.actor.kl_loss_coef=0.0 actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8192 actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=8192 actor_rollout_ref.rollout.enforce_eager=True reward.custom_reward_function.path=/run/reward_tolerant.py reward.custom_reward_function.name=compute_score ++trainer.val_before_train=True 'trainer.logger=["console","mlflow"]' trainer.default_local_dir=/checkpoints/char-count-qwen1.7b ++ python3 -c 'import torch_npu' ++ echo gpu + DEVICE=gpu + INFER_BACKEND=vllm + MODEL_PATH=/models/Qwen3-1.7B + TRAIN_FILE=/data/char_count/rl/train.parquet + TEST_FILE=/data/char_count/rl/test.parquet + NNODES=1 + NGPUS_PER_NODE=1 + TRAIN_BATCH_SIZE=128 + PPO_MINI_BATCH_SIZE=16 + PPO_MICRO_BATCH_SIZE_PER_GPU=8 + LOG_PROB_MICRO_BATCH_SIZE_PER_GPU=8 + MAX_PROMPT_LENGTH=128 + MAX_RESPONSE_LENGTH=128 + ACTOR_LR=1e-6 + KL_LOSS_COEF=0.001 + ENTROPY_COEFF=0 + ROLLOUT_TP=1 + ROLLOUT_GPU_MEM_UTIL=0.7 + ROLLOUT_N=8 + PROJECT_NAME=verl_char_count + EXPERIMENT_NAME=qwen3-1.7b-char-count-grpo-direct-1gpu + SAVE_FREQ=-1 + TEST_FREQ=5 + TOTAL_EPOCHS=2 + case "${DEVICE}" in + DATA=(algorithm.adv_estimator=grpo data.train_files=${TRAIN_FILE} data.val_files=${TEST_FILE} data.train_batch_size=${TRAIN_BATCH_SIZE} data.max_prompt_length=${MAX_PROMPT_LENGTH} data.max_response_length=${MAX_RESPONSE_LENGTH} data.filter_overlong_prompts=True data.truncation='error' algorithm.use_kl_in_reward=False) + MODEL=(actor_rollout_ref.model.path=${MODEL_PATH} actor_rollout_ref.model.use_remove_padding=True actor_rollout_ref.model.enable_gradient_checkpointing=True) + ACTOR=(actor_rollout_ref.actor.optim.lr=${ACTOR_LR} actor_rollout_ref.actor.ppo_mini_batch_size=${PPO_MINI_BATCH_SIZE} actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${PPO_MICRO_BATCH_SIZE_PER_GPU} actor_rollout_ref.actor.use_kl_loss=True actor_rollout_ref.actor.kl_loss_coef=${KL_LOSS_COEF} actor_rollout_ref.actor.kl_loss_type=low_var_kl actor_rollout_ref.actor.entropy_coeff=${ENTROPY_COEFF} actor_rollout_ref.actor.fsdp_config.param_offload=False actor_rollout_ref.actor.fsdp_config.optimizer_offload=False actor_rollout_ref.actor.ppo_max_token_len_per_gpu=3000 actor_rollout_ref.actor.use_dynamic_bsz=True) + ROLLOUT=(actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${LOG_PROB_MICRO_BATCH_SIZE_PER_GPU} actor_rollout_ref.rollout.tensor_model_parallel_size=${ROLLOUT_TP} actor_rollout_ref.rollout.name=${INFER_BACKEND} actor_rollout_ref.rollout.gpu_memory_utilization=${ROLLOUT_GPU_MEM_UTIL} actor_rollout_ref.rollout.enable_chunked_prefill=False actor_rollout_ref.rollout.enforce_eager=False actor_rollout_ref.rollout.free_cache_engine=True actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=4096 actor_rollout_ref.rollout.checkpoint_engine.update_weights_bucket_megabytes=4096 actor_rollout_ref.rollout.n=${ROLLOUT_N}) + REF=(actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=${LOG_PROB_MICRO_BATCH_SIZE_PER_GPU} actor_rollout_ref.ref.fsdp_config.param_offload=True actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192) + TRAINER=(trainer.critic_warmup=0 trainer.logger='["console","wandb"]' trainer.project_name=${PROJECT_NAME} trainer.experiment_name=${EXPERIMENT_NAME} trainer.n_gpus_per_node=${NGPUS_PER_NODE} trainer.nnodes=${NNODES} trainer.save_freq=${SAVE_FREQ} trainer.test_freq=${TEST_FREQ} trainer.total_epochs=${TOTAL_EPOCHS}) + EXTRA=() + LAUNCH=(python3) + RAY=(ray_kwargs.ray_init.runtime_env.py_executable=null) + '[' 1 '!=' 0 ']' + '[' gpu = gpu ']' + '[' vllm = vllm ']' + LAUNCH=(uv run --frozen --all-packages --extra "${INFER_BACKEND}" --extra fsdp python3) + RAY=(ray_kwargs.ray_init.runtime_env.py_executable="uv -v run --frozen --all-packages --extra ${INFER_BACKEND} --extra fsdp") + uv run --frozen --all-packages --extra vllm --extra fsdp python3 -m verl.trainer.main_ppo algorithm.adv_estimator=grpo data.train_files=/data/char_count/rl/train.parquet data.val_files=/data/char_count/rl/test.parquet data.train_batch_size=128 data.max_prompt_length=128 data.max_response_length=128 data.filter_overlong_prompts=True data.truncation=error algorithm.use_kl_in_reward=False actor_rollout_ref.model.path=/models/Qwen3-1.7B actor_rollout_ref.model.use_remove_padding=True actor_rollout_ref.model.enable_gradient_checkpointing=True actor_rollout_ref.actor.optim.lr=1e-6 actor_rollout_ref.actor.ppo_mini_batch_size=16 actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 actor_rollout_ref.actor.use_kl_loss=True actor_rollout_ref.actor.kl_loss_coef=0.001 actor_rollout_ref.actor.kl_loss_type=low_var_kl actor_rollout_ref.actor.entropy_coeff=0 actor_rollout_ref.actor.fsdp_config.param_offload=False actor_rollout_ref.actor.fsdp_config.optimizer_offload=False actor_rollout_ref.actor.ppo_max_token_len_per_gpu=3000 actor_rollout_ref.actor.use_dynamic_bsz=True actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 actor_rollout_ref.rollout.tensor_model_parallel_size=1 actor_rollout_ref.rollout.name=vllm actor_rollout_ref.rollout.gpu_memory_utilization=0.7 actor_rollout_ref.rollout.enable_chunked_prefill=False actor_rollout_ref.rollout.enforce_eager=False actor_rollout_ref.rollout.free_cache_engine=True actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=4096 actor_rollout_ref.rollout.checkpoint_engine.update_weights_bucket_megabytes=4096 actor_rollout_ref.rollout.n=8 actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 actor_rollout_ref.ref.fsdp_config.param_offload=True actor_rollout_ref.ref.log_prob_use_dynamic_bsz=True actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192 trainer.critic_warmup=0 'trainer.logger=["console","wandb"]' trainer.project_name=verl_char_count trainer.experiment_name=qwen3-1.7b-char-count-grpo-direct-1gpu trainer.n_gpus_per_node=1 trainer.nnodes=1 trainer.save_freq=-1 trainer.test_freq=5 trainer.total_epochs=2 'ray_kwargs.ray_init.runtime_env.py_executable=uv -v run --frozen --all-packages --extra vllm --extra fsdp' actor_rollout_ref.actor.strategy=fsdp2 actor_rollout_ref.actor.fsdp_config.param_offload=True actor_rollout_ref.actor.fsdp_config.optimizer_offload=True actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 '++data.apply_chat_template_kwargs={enable_thinking: false}' actor_rollout_ref.actor.use_kl_loss=False actor_rollout_ref.actor.kl_loss_coef=0.0 actor_rollout_ref.actor.ppo_max_token_len_per_gpu=8192 actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=8192 actor_rollout_ref.rollout.enforce_eager=True reward.custom_reward_function.path=/run/reward_tolerant.py reward.custom_reward_function.name=compute_score ++trainer.val_before_train=True 'trainer.logger=["console","mlflow"]' trainer.default_local_dir=/checkpoints/char-count-qwen1.7b /workspace/verl/verl/trainer/main_ppo.py:181: UserWarning: Disabled critic as algorithm.adv_estimator != gae. If it is not intended, please set critic.enable=True use_critic=need_critic(config), 2026-11-01 17:14:09,875 INFO worker.py:2003 -- Started a local Ray instance. View the dashboard at http://:8265 /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/worker.py:2051: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0 warnings.warn( (raylet) DEBUG Found workspace root: `/workspace/verl` (raylet) DEBUG Adding root workspace member: `/workspace/verl` (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) (raylet) DEBUG Found project root: `/workspace/verl` (raylet) DEBUG No workspace root found, using project root (raylet) DEBUG Discovered project `verl` at: /workspace/verl (raylet) DEBUG No Python version file found in workspace: /workspace/verl (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata (raylet) DEBUG Checking for Python environment at: `.venv` (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` (raylet) DEBUG Skipping environment synchronization due to `--no-sync` (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=41333 --object-store-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:41947 --session-name=session_2026-11-01_17-14-07_598925_196 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=3ae44e80d463b631d7fd94140af9fea5ec9e6078b42940dd11a74b17 --worker-id=26844d1302f67f929ee9aec69b27e4b565ad4e0095166abc8852b8b4 --worker-launch-time-ms=1793553250305 --node-id=96099fbef72585e71fedb24085e672ce36048668f7de1351ec53c223 --runtime-env-hash=247405026` (raylet) DEBUG Spawned child 2351 in process group 1 (TaskRunnerV1 pid=2351) WARNING:2026-11-01 17:14:14,128:Platform 'nvidia' (PlatformCUDA) is registered but not available. This may be due to this ray actor being a CPU-only actor. (TaskRunnerV1 pid=2351) /workspace/verl/verl/workers/engine/mindspeed/transformer_impl.py:24: UserWarning: NPU not support router replay for now. (TaskRunnerV1 pid=2351) from verl.utils.megatron.router_replay_patch import RouterReplay [validate_config] All configuration checks passed successfully! ray init kwargs: {'num_cpus': None, 'runtime_env': {'env_vars': {'TOKENIZERS_PARALLELISM': 'true', 'NCCL_DEBUG': 'WARN', 'VLLM_LOGGING_LEVEL': 'WARN', 'VLLM_ALLOW_RUNTIME_LORA_UPDATING': 'true', 'VLLM_DISABLE_COMPILE_CACHE': '1', 'HCCL_HOST_SOCKET_PORT_RANGE': 'auto', 'HCCL_NPU_SOCKET_PORT_RANGE': 'auto', 'HSA_NO_SCRATCH_RECLAIM': '1', 'PYTHONHASHSEED': '0', 'VERL_FULL_DETERMINISM': '0', 'VLLM_BATCH_INVARIANT': '0', 'VERL_RL_INSIGHT_ENABLE': '0'}, 'py_executable': 'uv -v run --frozen --all-packages --extra vllm --extra fsdp'}} (TaskRunnerV1 pid=2351) {'actor_rollout_ref': {'actor': {'_target_': 'verl.workers.config.FSDPActorConfig', (TaskRunnerV1 pid=2351) 'calculate_entropy': False, (TaskRunnerV1 pid=2351) 'calculate_sum_pi_squared': False, (TaskRunnerV1 pid=2351) 'checkpoint': {'_target_': 'verl.trainer.config.CheckpointConfig', (TaskRunnerV1 pid=2351) 'async_save': False, (TaskRunnerV1 pid=2351) 'load_contents': ['model', (TaskRunnerV1 pid=2351) 'optimizer', (TaskRunnerV1 pid=2351) 'extra'], (TaskRunnerV1 pid=2351) 'save_contents': ['model', (TaskRunnerV1 pid=2351) 'optimizer', (TaskRunnerV1 pid=2351) 'extra'], (TaskRunnerV1 pid=2351) 'strict': True}, (TaskRunnerV1 pid=2351) 'clip_ratio': 0.2, (TaskRunnerV1 pid=2351) 'clip_ratio_c': 3.0, (TaskRunnerV1 pid=2351) 'clip_ratio_high': 0.2, (TaskRunnerV1 pid=2351) 'clip_ratio_low': 0.2, (TaskRunnerV1 pid=2351) 'data_loader_seed': 42, (TaskRunnerV1 pid=2351) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2351) 'entropy_coeff': 0, (TaskRunnerV1 pid=2351) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2351) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2351) 'freeze_vision_tower': False, (TaskRunnerV1 pid=2351) 'fsdp_config': {'_target_': 'verl.workers.config.FSDPEngineConfig', (TaskRunnerV1 pid=2351) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2351) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2351) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2351) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2351) 'forward_only': False, (TaskRunnerV1 pid=2351) 'forward_prefetch': False, (TaskRunnerV1 pid=2351) 'fsdp_size': -1, (TaskRunnerV1 pid=2351) 'full_determinism': False, (TaskRunnerV1 pid=2351) 'model_dtype': 'bfloat16', (TaskRunnerV1 pid=2351) 'offload_policy': False, (TaskRunnerV1 pid=2351) 'optimizer_offload': True, (TaskRunnerV1 pid=2351) 'param_offload': True, (TaskRunnerV1 pid=2351) 'qat': {'_target_': 'verl.workers.config.QATEngineConfig', (TaskRunnerV1 pid=2351) 'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'group_size': 16, (TaskRunnerV1 pid=2351) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2351) 'embed_tokens', (TaskRunnerV1 pid=2351) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2351) 'mode': 'w4a16', (TaskRunnerV1 pid=2351) 'quantization_config_path': None}, (TaskRunnerV1 pid=2351) 'reshard_after_forward': True, (TaskRunnerV1 pid=2351) 'seed': 42, (TaskRunnerV1 pid=2351) 'strategy': 'fsdp', (TaskRunnerV1 pid=2351) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2351) 'use_orig_params': False, (TaskRunnerV1 pid=2351) 'use_torch_compile': True, (TaskRunnerV1 pid=2351) 'wrap_policy': {'min_num_params': 0}}, (TaskRunnerV1 pid=2351) 'grad_clip': 1.0, (TaskRunnerV1 pid=2351) 'kl_loss_coef': 0.0, (TaskRunnerV1 pid=2351) 'kl_loss_type': 'low_var_kl', (TaskRunnerV1 pid=2351) 'loss_agg_mode': 'token-mean', (TaskRunnerV1 pid=2351) 'loss_scale_factor': None, (TaskRunnerV1 pid=2351) 'optim': {'_target_': 'verl.workers.config.FSDPOptimizerConfig', (TaskRunnerV1 pid=2351) 'betas': [0.9, 0.999], (TaskRunnerV1 pid=2351) 'clip_grad': 1.0, (TaskRunnerV1 pid=2351) 'lr': 1e-06, (TaskRunnerV1 pid=2351) 'lr_scheduler_type': 'constant', (TaskRunnerV1 pid=2351) 'lr_warmup_steps': -1, (TaskRunnerV1 pid=2351) 'lr_warmup_steps_ratio': 0.0, (TaskRunnerV1 pid=2351) 'min_lr_ratio': 0.0, (TaskRunnerV1 pid=2351) 'num_cycles': 0.5, (TaskRunnerV1 pid=2351) 'optimizer': 'AdamW', (TaskRunnerV1 pid=2351) 'optimizer_impl': 'torch.optim', (TaskRunnerV1 pid=2351) 'override_optimizer_config': None, (TaskRunnerV1 pid=2351) 'total_training_steps': -1, (TaskRunnerV1 pid=2351) 'warmup_style': None, (TaskRunnerV1 pid=2351) 'weight_decay': 0.01, (TaskRunnerV1 pid=2351) 'zero_indexed_step': True}, (TaskRunnerV1 pid=2351) 'policy_loss': {'_target_': 'verl.workers.config.PolicyLossConfig', (TaskRunnerV1 pid=2351) 'clip_cov_lb': 1.0, (TaskRunnerV1 pid=2351) 'clip_cov_ratio': 0.0002, (TaskRunnerV1 pid=2351) 'clip_cov_ub': 5.0, (TaskRunnerV1 pid=2351) 'kl_cov_ratio': 0.0002, (TaskRunnerV1 pid=2351) 'loss_mode': 'vanilla', (TaskRunnerV1 pid=2351) 'ppo_kl_coef': 0.1}, (TaskRunnerV1 pid=2351) 'ppo_epochs': 1, (TaskRunnerV1 pid=2351) 'ppo_max_token_len_per_gpu': 8192, (TaskRunnerV1 pid=2351) 'ppo_micro_batch_size': None, (TaskRunnerV1 pid=2351) 'ppo_micro_batch_size_per_gpu': 8, (TaskRunnerV1 pid=2351) 'ppo_mini_batch_size': 16, (TaskRunnerV1 pid=2351) 'profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2351) 'all_ranks': False, (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'ranks': [], (TaskRunnerV1 pid=2351) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2351) 'tool': None, (TaskRunnerV1 pid=2351) 'tool_config': {'npu': {'_target_': 'verl.utils.profiler.config.NPUToolConfig', (TaskRunnerV1 pid=2351) 'analysis': True, (TaskRunnerV1 pid=2351) 'contents': [], (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'level': 'level0'}, (TaskRunnerV1 pid=2351) 'nsys': {'_target_': 'verl.utils.profiler.config.NsightToolConfig', (TaskRunnerV1 pid=2351) 'discrete': False}, (TaskRunnerV1 pid=2351) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2351) 'config_path': None, (TaskRunnerV1 pid=2351) 'stages': None, (TaskRunnerV1 pid=2351) 'steps': None, (TaskRunnerV1 pid=2351) 'strict': False}, (TaskRunnerV1 pid=2351) 'torch': {'_target_': 'verl.utils.profiler.config.TorchProfilerToolConfig', (TaskRunnerV1 pid=2351) 'contents': [], (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'schedule': {'_target_': 'verl.utils.profiler.config.TorchProfilerScheduleConfig', (TaskRunnerV1 pid=2351) 'active': 0, (TaskRunnerV1 pid=2351) 'repeat': 0, (TaskRunnerV1 pid=2351) 'skip_first': 0, (TaskRunnerV1 pid=2351) 'wait': 0, (TaskRunnerV1 pid=2351) 'warmup': 0}}, (TaskRunnerV1 pid=2351) 'torch_memory': {'_target_': 'verl.utils.profiler.config.TorchMemoryToolConfig', (TaskRunnerV1 pid=2351) 'stack_depth': 32, (TaskRunnerV1 pid=2351) 'trace_alloc_max_entries': 100000}}}, (TaskRunnerV1 pid=2351) 'qat': {'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'group_size': 16, (TaskRunnerV1 pid=2351) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2351) 'embed_tokens', (TaskRunnerV1 pid=2351) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2351) 'mode': 'w4a16', (TaskRunnerV1 pid=2351) 'quantization_config_path': None}, (TaskRunnerV1 pid=2351) 'rollout_n': 8, (TaskRunnerV1 pid=2351) 'router_replay': {'_target_': 'verl.workers.config.RouterReplayConfig', (TaskRunnerV1 pid=2351) 'mode': 'disabled', (TaskRunnerV1 pid=2351) 'record_file': None, (TaskRunnerV1 pid=2351) 'replay_file': None}, (TaskRunnerV1 pid=2351) 'shuffle': False, (TaskRunnerV1 pid=2351) 'strategy': 'fsdp2', (TaskRunnerV1 pid=2351) 'tau_neg': 1.05, (TaskRunnerV1 pid=2351) 'tau_pos': 1.0, (TaskRunnerV1 pid=2351) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2351) 'use_dynamic_bsz': True, (TaskRunnerV1 pid=2351) 'use_fused_kernels': False, (TaskRunnerV1 pid=2351) 'use_kl_loss': False, (TaskRunnerV1 pid=2351) 'use_prefix_grouper': False, (TaskRunnerV1 pid=2351) 'use_remove_padding': True, (TaskRunnerV1 pid=2351) 'use_torch_compile': True}, (TaskRunnerV1 pid=2351) 'hybrid_engine': True, (TaskRunnerV1 pid=2351) 'model': {'_target_': 'verl.workers.config.HFModelConfig', (TaskRunnerV1 pid=2351) 'custom_chat_template': None, (TaskRunnerV1 pid=2351) 'enable_activation_offload': False, (TaskRunnerV1 pid=2351) 'enable_gradient_checkpointing': True, (TaskRunnerV1 pid=2351) 'exclude_modules': None, (TaskRunnerV1 pid=2351) 'external_lib': None, (TaskRunnerV1 pid=2351) 'fused_kernel_options': {'impl_backend': 'torch'}, (TaskRunnerV1 pid=2351) 'hf_config_path': None, (TaskRunnerV1 pid=2351) 'lora': {'a2a_experimental': False, (TaskRunnerV1 pid=2351) 'adapter_path': None, (TaskRunnerV1 pid=2351) 'alpha': 32, (TaskRunnerV1 pid=2351) 'dropout': 0.0, (TaskRunnerV1 pid=2351) 'dropout_position': 'pre', (TaskRunnerV1 pid=2351) 'dtype': None, (TaskRunnerV1 pid=2351) 'exclude_modules': [], (TaskRunnerV1 pid=2351) 'freeze_language_model': True, (TaskRunnerV1 pid=2351) 'freeze_vision_model': True, (TaskRunnerV1 pid=2351) 'freeze_vision_projection': True, (TaskRunnerV1 pid=2351) 'lora_A_init_method': 'xavier', (TaskRunnerV1 pid=2351) 'lora_B_init_method': 'zero', (TaskRunnerV1 pid=2351) 'merge': False, (TaskRunnerV1 pid=2351) 'rank': 0, (TaskRunnerV1 pid=2351) 'target_modules': ['linear_qkv', (TaskRunnerV1 pid=2351) 'linear_proj', (TaskRunnerV1 pid=2351) 'linear_fc1', (TaskRunnerV1 pid=2351) 'linear_fc2'], (TaskRunnerV1 pid=2351) 'type': 'lora'}, (TaskRunnerV1 pid=2351) 'lora_adapter_path': None, (TaskRunnerV1 pid=2351) 'lora_alpha': 16, (TaskRunnerV1 pid=2351) 'lora_rank': 0, (TaskRunnerV1 pid=2351) 'mtp': {'_target_': 'verl.workers.config.MtpConfig', (TaskRunnerV1 pid=2351) 'detach_encoder': False, (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'enable_rollout': False, (TaskRunnerV1 pid=2351) 'enable_train': False, (TaskRunnerV1 pid=2351) 'method': 'mtp', (TaskRunnerV1 pid=2351) 'mtp_loss_scaling_factor': 0.1, (TaskRunnerV1 pid=2351) 'num_speculative_tokens': 1, (TaskRunnerV1 pid=2351) 'speculative_algorithm': 'EAGLE', (TaskRunnerV1 pid=2351) 'speculative_eagle_topk': 1, (TaskRunnerV1 pid=2351) 'speculative_num_draft_tokens': 4, (TaskRunnerV1 pid=2351) 'speculative_num_steps': 3}, (TaskRunnerV1 pid=2351) 'override_config': {}, (TaskRunnerV1 pid=2351) 'path': '/models/Qwen3-1.7B', (TaskRunnerV1 pid=2351) 'target_modules': 'all-linear', (TaskRunnerV1 pid=2351) 'tiled_mlp': {'enabled': False, (TaskRunnerV1 pid=2351) 'num_shards': 4}, (TaskRunnerV1 pid=2351) 'tokenizer_path': None, (TaskRunnerV1 pid=2351) 'trust_remote_code': False, (TaskRunnerV1 pid=2351) 'use_fused_kernels': False, (TaskRunnerV1 pid=2351) 'use_liger': False, (TaskRunnerV1 pid=2351) 'use_remove_padding': True, (TaskRunnerV1 pid=2351) 'use_shm': False}, (TaskRunnerV1 pid=2351) 'nccl_timeout': 600, (TaskRunnerV1 pid=2351) 'ref': {'_target_': 'verl.workers.config.FSDPActorConfig', (TaskRunnerV1 pid=2351) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2351) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2351) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2351) 'fsdp_config': {'_target_': 'verl.workers.config.FSDPEngineConfig', (TaskRunnerV1 pid=2351) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2351) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2351) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2351) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2351) 'forward_only': True, (TaskRunnerV1 pid=2351) 'forward_prefetch': False, (TaskRunnerV1 pid=2351) 'fsdp_size': -1, (TaskRunnerV1 pid=2351) 'full_determinism': False, (TaskRunnerV1 pid=2351) 'model_dtype': 'fp32', (TaskRunnerV1 pid=2351) 'offload_policy': False, (TaskRunnerV1 pid=2351) 'optimizer_offload': False, (TaskRunnerV1 pid=2351) 'param_offload': True, (TaskRunnerV1 pid=2351) 'qat': {'_target_': 'verl.workers.config.QATEngineConfig', (TaskRunnerV1 pid=2351) 'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'group_size': 16, (TaskRunnerV1 pid=2351) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2351) 'embed_tokens', (TaskRunnerV1 pid=2351) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2351) 'mode': 'w4a16', (TaskRunnerV1 pid=2351) 'quantization_config_path': None}, (TaskRunnerV1 pid=2351) 'reshard_after_forward': True, (TaskRunnerV1 pid=2351) 'seed': 42, (TaskRunnerV1 pid=2351) 'strategy': 'fsdp', (TaskRunnerV1 pid=2351) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2351) 'use_orig_params': False, (TaskRunnerV1 pid=2351) 'use_torch_compile': True, (TaskRunnerV1 pid=2351) 'wrap_policy': {'min_num_params': 0}}, (TaskRunnerV1 pid=2351) 'log_prob_max_token_len_per_gpu': 8192, (TaskRunnerV1 pid=2351) 'log_prob_micro_batch_size': None, (TaskRunnerV1 pid=2351) 'log_prob_micro_batch_size_per_gpu': 8, (TaskRunnerV1 pid=2351) 'log_prob_use_dynamic_bsz': True, (TaskRunnerV1 pid=2351) 'profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2351) 'all_ranks': False, (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'ranks': [], (TaskRunnerV1 pid=2351) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2351) 'tool': None, (TaskRunnerV1 pid=2351) 'tool_config': {'npu': {'_target_': 'verl.utils.profiler.config.NPUToolConfig', (TaskRunnerV1 pid=2351) 'analysis': True, (TaskRunnerV1 pid=2351) 'contents': [], (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'level': 'level0'}, (TaskRunnerV1 pid=2351) 'nsys': {'_target_': 'verl.utils.profiler.config.NsightToolConfig', (TaskRunnerV1 pid=2351) 'discrete': False}, (TaskRunnerV1 pid=2351) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2351) 'config_path': None, (TaskRunnerV1 pid=2351) 'stages': None, (TaskRunnerV1 pid=2351) 'steps': None, (TaskRunnerV1 pid=2351) 'strict': False}, (TaskRunnerV1 pid=2351) 'torch': {'_target_': 'verl.utils.profiler.config.TorchProfilerToolConfig', (TaskRunnerV1 pid=2351) 'contents': [], (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'schedule': {'_target_': 'verl.utils.profiler.config.TorchProfilerScheduleConfig', (TaskRunnerV1 pid=2351) 'active': 0, (TaskRunnerV1 pid=2351) 'repeat': 0, (TaskRunnerV1 pid=2351) 'skip_first': 0, (TaskRunnerV1 pid=2351) 'wait': 0, (TaskRunnerV1 pid=2351) 'warmup': 0}}, (TaskRunnerV1 pid=2351) 'torch_memory': {'_target_': 'verl.utils.profiler.config.TorchMemoryToolConfig', (TaskRunnerV1 pid=2351) 'stack_depth': 32, (TaskRunnerV1 pid=2351) 'trace_alloc_max_entries': 100000}}}, (TaskRunnerV1 pid=2351) 'rollout_n': 8, (TaskRunnerV1 pid=2351) 'router_replay': {'_target_': 'verl.workers.config.RouterReplayConfig', (TaskRunnerV1 pid=2351) 'mode': 'disabled', (TaskRunnerV1 pid=2351) 'record_file': None, (TaskRunnerV1 pid=2351) 'replay_file': None}, (TaskRunnerV1 pid=2351) 'strategy': 'fsdp2', (TaskRunnerV1 pid=2351) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2351) 'use_torch_compile': True}, (TaskRunnerV1 pid=2351) 'rollout': {'_target_': 'verl.workers.config.RolloutConfig', (TaskRunnerV1 pid=2351) 'agent': {'_target_': 'verl.workers.config.AgentLoopConfig', (TaskRunnerV1 pid=2351) 'agent_loop_config_path': None, (TaskRunnerV1 pid=2351) 'custom_async_server': {'_target_': 'verl.workers.config.CustomAsyncServerConfig', (TaskRunnerV1 pid=2351) 'name': None, (TaskRunnerV1 pid=2351) 'path': None}, (TaskRunnerV1 pid=2351) 'default_agent_loop': 'single_turn_agent', (TaskRunnerV1 pid=2351) 'num_workers': 8}, (TaskRunnerV1 pid=2351) 'calculate_log_probs': True, (TaskRunnerV1 pid=2351) 'checkpoint_engine': {'_target_': 'verl.workers.config.CheckpointEngineConfig', (TaskRunnerV1 pid=2351) 'backend': 'naive', (TaskRunnerV1 pid=2351) 'custom_backend_module': None, (TaskRunnerV1 pid=2351) 'engine_kwargs': {}, (TaskRunnerV1 pid=2351) 'update_weights_bucket_megabytes': 4096}, (TaskRunnerV1 pid=2351) 'cudagraph_capture_sizes': None, (TaskRunnerV1 pid=2351) 'data_parallel_size': 1, (TaskRunnerV1 pid=2351) 'disable_log_stats': True, (TaskRunnerV1 pid=2351) 'disaggregation': {'bootstrap_port': None, (TaskRunnerV1 pid=2351) 'decode_replicas': 1, (TaskRunnerV1 pid=2351) 'decode_tensor_model_parallel_size': None, (TaskRunnerV1 pid=2351) 'enabled': False, (TaskRunnerV1 pid=2351) 'ib_device': None, (TaskRunnerV1 pid=2351) 'prefill_replicas': 1, (TaskRunnerV1 pid=2351) 'transfer_backend': 'nixl'}, (TaskRunnerV1 pid=2351) 'do_sample': True, (TaskRunnerV1 pid=2351) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2351) 'enable_chunked_prefill': False, (TaskRunnerV1 pid=2351) 'enable_prefix_caching': True, (TaskRunnerV1 pid=2351) 'enable_rollout_routing_replay': False, (TaskRunnerV1 pid=2351) 'enforce_eager': True, (TaskRunnerV1 pid=2351) 'engine_kwargs': {'sglang': {}, (TaskRunnerV1 pid=2351) 'trtllm': {}, (TaskRunnerV1 pid=2351) 'vllm': {}}, (TaskRunnerV1 pid=2351) 'expert_parallel_size': 1, (TaskRunnerV1 pid=2351) 'free_cache_engine': True, (TaskRunnerV1 pid=2351) 'full_determinism': False, (TaskRunnerV1 pid=2351) 'gpu_memory_utilization': 0.7, (TaskRunnerV1 pid=2351) 'ignore_eos': False, (TaskRunnerV1 pid=2351) 'layered_summon': False, (TaskRunnerV1 pid=2351) 'load_format': 'dummy', (TaskRunnerV1 pid=2351) 'log_prob_max_token_len_per_gpu': 8192, (TaskRunnerV1 pid=2351) 'log_prob_micro_batch_size': None, (TaskRunnerV1 pid=2351) 'log_prob_micro_batch_size_per_gpu': 8, (TaskRunnerV1 pid=2351) 'log_prob_use_dynamic_bsz': True, (TaskRunnerV1 pid=2351) 'logprobs_mode': 'processed_logprobs', (TaskRunnerV1 pid=2351) 'max_model_len': None, (TaskRunnerV1 pid=2351) 'max_num_batched_tokens': 8192, (TaskRunnerV1 pid=2351) 'max_num_seqs': 1024, (TaskRunnerV1 pid=2351) 'mode': 'async', (TaskRunnerV1 pid=2351) 'moe_load_balance_metrics_interval': 0, (TaskRunnerV1 pid=2351) 'mtp': {'_target_': 'verl.workers.config.MtpConfig', (TaskRunnerV1 pid=2351) 'detach_encoder': False, (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'enable_rollout': False, (TaskRunnerV1 pid=2351) 'enable_train': False, (TaskRunnerV1 pid=2351) 'method': 'mtp', (TaskRunnerV1 pid=2351) 'mtp_loss_scaling_factor': 0.1, (TaskRunnerV1 pid=2351) 'num_speculative_tokens': 1, (TaskRunnerV1 pid=2351) 'speculative_algorithm': 'EAGLE', (TaskRunnerV1 pid=2351) 'speculative_eagle_topk': 1, (TaskRunnerV1 pid=2351) 'speculative_num_draft_tokens': 4, (TaskRunnerV1 pid=2351) 'speculative_num_steps': 3}, (TaskRunnerV1 pid=2351) 'multi_stage_wake_up': False, (TaskRunnerV1 pid=2351) 'multi_turn': {'_target_': 'verl.workers.config.MultiTurnConfig', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'format': 'hermes', (TaskRunnerV1 pid=2351) 'function_tool_path': None, (TaskRunnerV1 pid=2351) 'max_assistant_turns': None, (TaskRunnerV1 pid=2351) 'max_parallel_calls': 1, (TaskRunnerV1 pid=2351) 'max_tool_response_length': 256, (TaskRunnerV1 pid=2351) 'max_user_turns': None, (TaskRunnerV1 pid=2351) 'num_repeat_rollouts': None, (TaskRunnerV1 pid=2351) 'tokenization_sanity_check_mode': 'strict', (TaskRunnerV1 pid=2351) 'tool_config_path': None, (TaskRunnerV1 pid=2351) 'tool_response_truncate_side': 'middle', (TaskRunnerV1 pid=2351) 'use_inference_chat_template': False}, (TaskRunnerV1 pid=2351) 'n': 8, (TaskRunnerV1 pid=2351) 'n_gpus_per_node': 1, (TaskRunnerV1 pid=2351) 'name': 'vllm', (TaskRunnerV1 pid=2351) 'nnodes': 0, (TaskRunnerV1 pid=2351) 'over_sample_rate': 0, (TaskRunnerV1 pid=2351) 'pipeline_model_parallel_size': 1, (TaskRunnerV1 pid=2351) 'profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2351) 'all_ranks': False, (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'ranks': [], (TaskRunnerV1 pid=2351) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2351) 'tool': None, (TaskRunnerV1 pid=2351) 'tool_config': {'npu': {'_target_': 'verl.utils.profiler.config.NPUToolConfig', (TaskRunnerV1 pid=2351) 'analysis': True, (TaskRunnerV1 pid=2351) 'contents': [], (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'level': 'level0', (TaskRunnerV1 pid=2351) 'profile_token_end': None, (TaskRunnerV1 pid=2351) 'profile_token_start': None}, (TaskRunnerV1 pid=2351) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2351) 'config_path': None, (TaskRunnerV1 pid=2351) 'stages': None, (TaskRunnerV1 pid=2351) 'steps': None, (TaskRunnerV1 pid=2351) 'strict': False}, (TaskRunnerV1 pid=2351) 'torch': {'_target_': 'verl.utils.profiler.config.TorchProfilerToolConfig', (TaskRunnerV1 pid=2351) 'contents': [], (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'profile_token_end': None, (TaskRunnerV1 pid=2351) 'profile_token_start': None}}}, (TaskRunnerV1 pid=2351) 'prometheus': {'_target_': 'verl.workers.config.PrometheusConfig', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'file': '/tmp/ray/session_latest/metrics/prometheus/prometheus.yml', (TaskRunnerV1 pid=2351) 'port': 9090, (TaskRunnerV1 pid=2351) 'served_model_name': '/models/Qwen3-1.7B'}, (TaskRunnerV1 pid=2351) 'prompt_length': 128, (TaskRunnerV1 pid=2351) 'qat': {'_target_': 'verl.workers.config.QATEngineConfig', (TaskRunnerV1 pid=2351) 'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'group_size': 16, (TaskRunnerV1 pid=2351) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2351) 'embed_tokens', (TaskRunnerV1 pid=2351) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2351) 'mode': 'w4a16', (TaskRunnerV1 pid=2351) 'quantization_config_path': None}, (TaskRunnerV1 pid=2351) 'quantization': None, (TaskRunnerV1 pid=2351) 'quantization_config_file': None, (TaskRunnerV1 pid=2351) 'response_length': 128, (TaskRunnerV1 pid=2351) 'scheduling_policy': 'fcfs', (TaskRunnerV1 pid=2351) 'seed': 42, (TaskRunnerV1 pid=2351) 'skip_tokenizer_init': True, (TaskRunnerV1 pid=2351) 'temperature': 1.0, (TaskRunnerV1 pid=2351) 'tensor_model_parallel_size': 1, (TaskRunnerV1 pid=2351) 'top_k': -1, (TaskRunnerV1 pid=2351) 'top_p': 1, (TaskRunnerV1 pid=2351) 'trace': {'_target_': 'verl.workers.config.TraceConfig', (TaskRunnerV1 pid=2351) 'backend': None, (TaskRunnerV1 pid=2351) 'experiment_name': 'qwen3-1.7b-char-count-grpo-direct-1gpu', (TaskRunnerV1 pid=2351) 'max_samples_per_step_per_worker': None, (TaskRunnerV1 pid=2351) 'project_name': 'verl_char_count', (TaskRunnerV1 pid=2351) 'token2text': False}, (TaskRunnerV1 pid=2351) 'val_kwargs': {'_target_': 'verl.workers.config.SamplingConfig', (TaskRunnerV1 pid=2351) 'do_sample': False, (TaskRunnerV1 pid=2351) 'n': 1, (TaskRunnerV1 pid=2351) 'temperature': 0, (TaskRunnerV1 pid=2351) 'top_k': -1, (TaskRunnerV1 pid=2351) 'top_p': 1.0}}}, (TaskRunnerV1 pid=2351) 'algorithm': {'_target_': 'verl.trainer.config.AlgoConfig', (TaskRunnerV1 pid=2351) 'adv_estimator': 'grpo', (TaskRunnerV1 pid=2351) 'filter_groups': {'_target_': 'verl.trainer.config.FilterGroupsConfig', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'max_inflight_gen_batches': 1, (TaskRunnerV1 pid=2351) 'max_num_gen_batches': 0, (TaskRunnerV1 pid=2351) 'metric': None}, (TaskRunnerV1 pid=2351) 'gamma': 1.0, (TaskRunnerV1 pid=2351) 'kl_ctrl': {'_target_': 'verl.trainer.config.KLControlConfig', (TaskRunnerV1 pid=2351) 'horizon': 10000, (TaskRunnerV1 pid=2351) 'kl_coef': 0.001, (TaskRunnerV1 pid=2351) 'target_kl': 0.1, (TaskRunnerV1 pid=2351) 'type': 'fixed'}, (TaskRunnerV1 pid=2351) 'kl_penalty': 'kl', (TaskRunnerV1 pid=2351) 'lam': 1.0, (TaskRunnerV1 pid=2351) 'norm_adv_by_std_in_grpo': True, (TaskRunnerV1 pid=2351) 'pf_ppo': {'reweight_method': 'pow', 'weight_pow': 2.0}, (TaskRunnerV1 pid=2351) 'rollout_correction': {'bypass_mode': False, (TaskRunnerV1 pid=2351) 'loss_type': 'ppo_clip', (TaskRunnerV1 pid=2351) 'rollout_is': None, (TaskRunnerV1 pid=2351) 'rollout_is_batch_normalize': False, (TaskRunnerV1 pid=2351) 'rollout_is_threshold': 2.0, (TaskRunnerV1 pid=2351) 'rollout_rs': None, (TaskRunnerV1 pid=2351) 'rollout_rs_threshold': None}, (TaskRunnerV1 pid=2351) 'use_kl_in_reward': False, (TaskRunnerV1 pid=2351) 'use_pf_ppo': False}, (TaskRunnerV1 pid=2351) 'critic': {'_target_': 'verl.workers.config.FSDPCriticConfig', (TaskRunnerV1 pid=2351) 'checkpoint': {'_target_': 'verl.trainer.config.CheckpointConfig', (TaskRunnerV1 pid=2351) 'async_save': False, (TaskRunnerV1 pid=2351) 'load_contents': ['model', 'optimizer', 'extra'], (TaskRunnerV1 pid=2351) 'save_contents': ['model', 'optimizer', 'extra']}, (TaskRunnerV1 pid=2351) 'cliprange_value': 0.5, (TaskRunnerV1 pid=2351) 'data_loader_seed': 42, (TaskRunnerV1 pid=2351) 'enable': None, (TaskRunnerV1 pid=2351) 'forward_max_token_len_per_gpu': 32768, (TaskRunnerV1 pid=2351) 'forward_micro_batch_size': None, (TaskRunnerV1 pid=2351) 'forward_micro_batch_size_per_gpu': None, (TaskRunnerV1 pid=2351) 'fsdp': {'_target_': 'verl.workers.config.FSDPEngineConfig', (TaskRunnerV1 pid=2351) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2351) 'entropy_checkpointing': False, (TaskRunnerV1 pid=2351) 'entropy_from_logits_chunk_size': 2048, (TaskRunnerV1 pid=2351) 'entropy_from_logits_with_chunking': False, (TaskRunnerV1 pid=2351) 'forward_only': False, (TaskRunnerV1 pid=2351) 'forward_prefetch': False, (TaskRunnerV1 pid=2351) 'fsdp_size': -1, (TaskRunnerV1 pid=2351) 'full_determinism': False, (TaskRunnerV1 pid=2351) 'model_dtype': 'fp32', (TaskRunnerV1 pid=2351) 'offload_policy': False, (TaskRunnerV1 pid=2351) 'optimizer_offload': False, (TaskRunnerV1 pid=2351) 'param_offload': False, (TaskRunnerV1 pid=2351) 'qat': {'_target_': 'verl.workers.config.QATEngineConfig', (TaskRunnerV1 pid=2351) 'activation_observer': 'static_minmax', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'group_size': 16, (TaskRunnerV1 pid=2351) 'ignore_patterns': ['lm_head', (TaskRunnerV1 pid=2351) 'embed_tokens', (TaskRunnerV1 pid=2351) 're:.*mlp.gate$'], (TaskRunnerV1 pid=2351) 'mode': 'w4a16', (TaskRunnerV1 pid=2351) 'quantization_config_path': None}, (TaskRunnerV1 pid=2351) 'reshard_after_forward': True, (TaskRunnerV1 pid=2351) 'seed': 42, (TaskRunnerV1 pid=2351) 'strategy': 'fsdp', (TaskRunnerV1 pid=2351) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2351) 'use_orig_params': False, (TaskRunnerV1 pid=2351) 'use_torch_compile': True, (TaskRunnerV1 pid=2351) 'wrap_policy': {'min_num_params': 0}}, (TaskRunnerV1 pid=2351) 'grad_clip': 1.0, (TaskRunnerV1 pid=2351) 'loss_agg_mode': 'token-mean', (TaskRunnerV1 pid=2351) 'loss_scale_factor': None, (TaskRunnerV1 pid=2351) 'model': {'_target_': 'verl.workers.config.HFModelConfig', (TaskRunnerV1 pid=2351) 'custom_chat_template': None, (TaskRunnerV1 pid=2351) 'enable_activation_offload': False, (TaskRunnerV1 pid=2351) 'enable_gradient_checkpointing': True, (TaskRunnerV1 pid=2351) 'exclude_modules': None, (TaskRunnerV1 pid=2351) 'external_lib': None, (TaskRunnerV1 pid=2351) 'fused_kernel_options': {'impl_backend': 'torch'}, (TaskRunnerV1 pid=2351) 'hf_config_path': None, (TaskRunnerV1 pid=2351) 'lora': {'a2a_experimental': False, (TaskRunnerV1 pid=2351) 'adapter_path': None, (TaskRunnerV1 pid=2351) 'alpha': 32, (TaskRunnerV1 pid=2351) 'dropout': 0.0, (TaskRunnerV1 pid=2351) 'dropout_position': 'pre', (TaskRunnerV1 pid=2351) 'dtype': None, (TaskRunnerV1 pid=2351) 'exclude_modules': [], (TaskRunnerV1 pid=2351) 'freeze_language_model': True, (TaskRunnerV1 pid=2351) 'freeze_vision_model': True, (TaskRunnerV1 pid=2351) 'freeze_vision_projection': True, (TaskRunnerV1 pid=2351) 'lora_A_init_method': 'xavier', (TaskRunnerV1 pid=2351) 'lora_B_init_method': 'zero', (TaskRunnerV1 pid=2351) 'merge': False, (TaskRunnerV1 pid=2351) 'rank': 0, (TaskRunnerV1 pid=2351) 'target_modules': ['linear_qkv', (TaskRunnerV1 pid=2351) 'linear_proj', (TaskRunnerV1 pid=2351) 'linear_fc1', (TaskRunnerV1 pid=2351) 'linear_fc2'], (TaskRunnerV1 pid=2351) 'type': 'lora'}, (TaskRunnerV1 pid=2351) 'lora_adapter_path': None, (TaskRunnerV1 pid=2351) 'lora_alpha': 16, (TaskRunnerV1 pid=2351) 'lora_rank': 0, (TaskRunnerV1 pid=2351) 'mtp': {'_target_': 'verl.workers.config.MtpConfig', (TaskRunnerV1 pid=2351) 'detach_encoder': False, (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'enable_rollout': False, (TaskRunnerV1 pid=2351) 'enable_train': False, (TaskRunnerV1 pid=2351) 'method': 'mtp', (TaskRunnerV1 pid=2351) 'mtp_loss_scaling_factor': 0.1, (TaskRunnerV1 pid=2351) 'num_speculative_tokens': 1, (TaskRunnerV1 pid=2351) 'speculative_algorithm': 'EAGLE', (TaskRunnerV1 pid=2351) 'speculative_eagle_topk': 1, (TaskRunnerV1 pid=2351) 'speculative_num_draft_tokens': 4, (TaskRunnerV1 pid=2351) 'speculative_num_steps': 3}, (TaskRunnerV1 pid=2351) 'override_config': {}, (TaskRunnerV1 pid=2351) 'path': '~/models/deepseek-llm-7b-chat', (TaskRunnerV1 pid=2351) 'target_modules': 'all-linear', (TaskRunnerV1 pid=2351) 'tiled_mlp': {'enabled': False, 'num_shards': 4}, (TaskRunnerV1 pid=2351) 'tokenizer_path': None, (TaskRunnerV1 pid=2351) 'trust_remote_code': False, (TaskRunnerV1 pid=2351) 'use_fused_kernels': False, (TaskRunnerV1 pid=2351) 'use_liger': False, (TaskRunnerV1 pid=2351) 'use_remove_padding': True, (TaskRunnerV1 pid=2351) 'use_shm': False}, (TaskRunnerV1 pid=2351) 'optim': {'_target_': 'verl.workers.config.FSDPOptimizerConfig', (TaskRunnerV1 pid=2351) 'betas': [0.9, 0.999], (TaskRunnerV1 pid=2351) 'clip_grad': 1.0, (TaskRunnerV1 pid=2351) 'lr': 1e-05, (TaskRunnerV1 pid=2351) 'lr_scheduler_type': 'constant', (TaskRunnerV1 pid=2351) 'lr_warmup_steps': -1, (TaskRunnerV1 pid=2351) 'lr_warmup_steps_ratio': 0.0, (TaskRunnerV1 pid=2351) 'min_lr_ratio': 0.0, (TaskRunnerV1 pid=2351) 'num_cycles': 0.5, (TaskRunnerV1 pid=2351) 'optimizer': 'AdamW', (TaskRunnerV1 pid=2351) 'optimizer_impl': 'torch.optim', (TaskRunnerV1 pid=2351) 'override_optimizer_config': None, (TaskRunnerV1 pid=2351) 'total_training_steps': -1, (TaskRunnerV1 pid=2351) 'warmup_style': None, (TaskRunnerV1 pid=2351) 'weight_decay': 0.01, (TaskRunnerV1 pid=2351) 'zero_indexed_step': True}, (TaskRunnerV1 pid=2351) 'ppo_epochs': 1, (TaskRunnerV1 pid=2351) 'ppo_max_token_len_per_gpu': 32768, (TaskRunnerV1 pid=2351) 'ppo_micro_batch_size': None, (TaskRunnerV1 pid=2351) 'ppo_micro_batch_size_per_gpu': None, (TaskRunnerV1 pid=2351) 'ppo_mini_batch_size': 16, (TaskRunnerV1 pid=2351) 'profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2351) 'all_ranks': False, (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'ranks': [], (TaskRunnerV1 pid=2351) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2351) 'tool': None, (TaskRunnerV1 pid=2351) 'tool_config': {'npu': {'_target_': 'verl.utils.profiler.config.NPUToolConfig', (TaskRunnerV1 pid=2351) 'analysis': True, (TaskRunnerV1 pid=2351) 'contents': [], (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'level': 'level0'}, (TaskRunnerV1 pid=2351) 'nsys': {'_target_': 'verl.utils.profiler.config.NsightToolConfig', (TaskRunnerV1 pid=2351) 'discrete': False}, (TaskRunnerV1 pid=2351) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2351) 'config_path': None, (TaskRunnerV1 pid=2351) 'stages': None, (TaskRunnerV1 pid=2351) 'steps': None, (TaskRunnerV1 pid=2351) 'strict': False}, (TaskRunnerV1 pid=2351) 'torch': {'_target_': 'verl.utils.profiler.config.TorchProfilerToolConfig', (TaskRunnerV1 pid=2351) 'contents': [], (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'schedule': {'_target_': 'verl.utils.profiler.config.TorchProfilerScheduleConfig', (TaskRunnerV1 pid=2351) 'active': 0, (TaskRunnerV1 pid=2351) 'repeat': 0, (TaskRunnerV1 pid=2351) 'skip_first': 0, (TaskRunnerV1 pid=2351) 'wait': 0, (TaskRunnerV1 pid=2351) 'warmup': 0}}, (TaskRunnerV1 pid=2351) 'torch_memory': {'_target_': 'verl.utils.profiler.config.TorchMemoryToolConfig', (TaskRunnerV1 pid=2351) 'stack_depth': 32, (TaskRunnerV1 pid=2351) 'trace_alloc_max_entries': 100000}}}, (TaskRunnerV1 pid=2351) 'rollout_n': 8, (TaskRunnerV1 pid=2351) 'shuffle': False, (TaskRunnerV1 pid=2351) 'strategy': 'fsdp', (TaskRunnerV1 pid=2351) 'ulysses_sequence_parallel_size': 1, (TaskRunnerV1 pid=2351) 'use_dynamic_bsz': True}, (TaskRunnerV1 pid=2351) 'custom_reward_function': {'name': None, 'path': None}, (TaskRunnerV1 pid=2351) 'data': {'apply_chat_template_kwargs': {'enable_thinking': False}, (TaskRunnerV1 pid=2351) 'audio_key': 'audios', (TaskRunnerV1 pid=2351) 'continuous_token': {'enable': False, 'model_family': 'auto'}, (TaskRunnerV1 pid=2351) 'custom_cls': {'name': None, 'path': None}, (TaskRunnerV1 pid=2351) 'dataloader_num_workers': 8, (TaskRunnerV1 pid=2351) 'filter_overlong_prompts': True, (TaskRunnerV1 pid=2351) 'filter_overlong_prompts_workers': 1, (TaskRunnerV1 pid=2351) 'function_tool_path': None, (TaskRunnerV1 pid=2351) 'gen_batch_size': None, (TaskRunnerV1 pid=2351) 'image_key': 'images', (TaskRunnerV1 pid=2351) 'image_patch_size': 14, (TaskRunnerV1 pid=2351) 'max_prompt_length': 128, (TaskRunnerV1 pid=2351) 'max_response_length': 128, (TaskRunnerV1 pid=2351) 'mm_processor_kwargs': {}, (TaskRunnerV1 pid=2351) 'prompt_key': 'prompt', (TaskRunnerV1 pid=2351) 'return_full_prompt': False, (TaskRunnerV1 pid=2351) 'return_multi_modal_inputs': True, (TaskRunnerV1 pid=2351) 'return_raw_chat': True, (TaskRunnerV1 pid=2351) 'return_raw_input_ids': False, (TaskRunnerV1 pid=2351) 'reward_fn_key': 'data_source', (TaskRunnerV1 pid=2351) 'seed': None, (TaskRunnerV1 pid=2351) 'shuffle': True, (TaskRunnerV1 pid=2351) 'tokenizer': None, (TaskRunnerV1 pid=2351) 'tool_config_path': None, (TaskRunnerV1 pid=2351) 'train_batch_size': 128, (TaskRunnerV1 pid=2351) 'train_files': '/data/char_count/rl/train.parquet', (TaskRunnerV1 pid=2351) 'train_max_samples': -1, (TaskRunnerV1 pid=2351) 'truncation': 'error', (TaskRunnerV1 pid=2351) 'trust_remote_code': False, (TaskRunnerV1 pid=2351) 'use_shm': False, (TaskRunnerV1 pid=2351) 'val_batch_size': None, (TaskRunnerV1 pid=2351) 'val_files': '/data/char_count/rl/test.parquet', (TaskRunnerV1 pid=2351) 'val_max_samples': -1, (TaskRunnerV1 pid=2351) 'validation_shuffle': False, (TaskRunnerV1 pid=2351) 'video_key': 'videos'}, (TaskRunnerV1 pid=2351) 'distillation': {'_target_': 'verl.workers.config.DistillationConfig', (TaskRunnerV1 pid=2351) 'distillation_loss': {'_target_': 'verl.workers.config.DistillationLossConfig', (TaskRunnerV1 pid=2351) 'clip_ratio': 0.2, (TaskRunnerV1 pid=2351) 'clip_ratio_high': 0.2, (TaskRunnerV1 pid=2351) 'clip_ratio_low': 0.2, (TaskRunnerV1 pid=2351) 'distillation_loss_coef': 1.0, (TaskRunnerV1 pid=2351) 'log_prob_min_clamp': None, (TaskRunnerV1 pid=2351) 'loss_max_clamp': None, (TaskRunnerV1 pid=2351) 'loss_mode': 'k3', (TaskRunnerV1 pid=2351) 'policy_loss_mode': 'vanilla', (TaskRunnerV1 pid=2351) 'topk': 32, (TaskRunnerV1 pid=2351) 'use_policy_gradient': False, (TaskRunnerV1 pid=2351) 'use_task_rewards': True}, (TaskRunnerV1 pid=2351) 'enabled': False, (TaskRunnerV1 pid=2351) 'n_gpus_per_node': 8, (TaskRunnerV1 pid=2351) 'nnodes': 0, (TaskRunnerV1 pid=2351) 'teacher_key': 'data_source', (TaskRunnerV1 pid=2351) 'teacher_models': {'teacher_model': {'_target_': 'verl.workers.config.DistillationTeacherModelConfig', (TaskRunnerV1 pid=2351) 'inference': {'_target_': 'verl.workers.config.RolloutConfig', (TaskRunnerV1 pid=2351) 'cudagraph_capture_sizes': None, (TaskRunnerV1 pid=2351) 'data_parallel_size': 1, (TaskRunnerV1 pid=2351) 'disable_log_stats': True, (TaskRunnerV1 pid=2351) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2351) 'enable_chunked_prefill': True, (TaskRunnerV1 pid=2351) 'enable_prefix_caching': True, (TaskRunnerV1 pid=2351) 'enforce_eager': True, (TaskRunnerV1 pid=2351) 'engine_kwargs': {}, (TaskRunnerV1 pid=2351) 'expert_parallel_size': 1, (TaskRunnerV1 pid=2351) 'free_cache_engine': True, (TaskRunnerV1 pid=2351) 'gpu_memory_utilization': 0.5, (TaskRunnerV1 pid=2351) 'limit_images': None, (TaskRunnerV1 pid=2351) 'load_format': 'auto', (TaskRunnerV1 pid=2351) 'max_model_len': None, (TaskRunnerV1 pid=2351) 'max_num_batched_tokens': 8192, (TaskRunnerV1 pid=2351) 'max_num_seqs': 1024, (TaskRunnerV1 pid=2351) 'name': 'vllm', (TaskRunnerV1 pid=2351) 'prompt_length': 128, (TaskRunnerV1 pid=2351) 'response_length': 128, (TaskRunnerV1 pid=2351) 'skip_tokenizer_init': True, (TaskRunnerV1 pid=2351) 'temperature': 1.0, (TaskRunnerV1 pid=2351) 'tensor_model_parallel_size': 2}, (TaskRunnerV1 pid=2351) 'key': None, (TaskRunnerV1 pid=2351) 'model_path': None, (TaskRunnerV1 pid=2351) 'num_replicas': 0}}}, (TaskRunnerV1 pid=2351) 'global_profiler': {'_target_': 'verl.utils.profiler.ProfilerConfig', (TaskRunnerV1 pid=2351) 'global_tool_config': {'nsys': {'_target_': 'verl.utils.profiler.config.NsightToolConfig', (TaskRunnerV1 pid=2351) 'controller_nsight_options': {'cuda-graph-trace': 'graph', (TaskRunnerV1 pid=2351) 'cuda-memory-usage': 'true', (TaskRunnerV1 pid=2351) 'trace': 'cuda,nvtx,cublas,ucx'}, (TaskRunnerV1 pid=2351) 'discrete': False, (TaskRunnerV1 pid=2351) 'worker_nsight_options': {'capture-range': 'cudaProfilerApi', (TaskRunnerV1 pid=2351) 'capture-range-end': None, (TaskRunnerV1 pid=2351) 'cuda-graph-trace': 'graph', (TaskRunnerV1 pid=2351) 'cuda-memory-usage': 'true', (TaskRunnerV1 pid=2351) 'kill': 'none', (TaskRunnerV1 pid=2351) 'trace': 'cuda,nvtx,cublas,ucx'}}, (TaskRunnerV1 pid=2351) 'precision_debugger': {'_target_': 'verl.utils.profiler.config.PrecisionDebuggerToolConfig', (TaskRunnerV1 pid=2351) 'config_path': None, (TaskRunnerV1 pid=2351) 'stages': None, (TaskRunnerV1 pid=2351) 'steps': None, (TaskRunnerV1 pid=2351) 'strict': False}, (TaskRunnerV1 pid=2351) 'torch_memory': {'context': 'all', (TaskRunnerV1 pid=2351) 'kw_args': {}, (TaskRunnerV1 pid=2351) 'stack_depth': 32, (TaskRunnerV1 pid=2351) 'stacks': 'all', (TaskRunnerV1 pid=2351) 'trace_alloc_max_entries': 100000}}, (TaskRunnerV1 pid=2351) 'profile_continuous_steps': False, (TaskRunnerV1 pid=2351) 'save_path': 'outputs/profile', (TaskRunnerV1 pid=2351) 'steps': None, (TaskRunnerV1 pid=2351) 'tool': None}, (TaskRunnerV1 pid=2351) 'model_engine': 'dp', (TaskRunnerV1 pid=2351) 'ray_kwargs': {'ray_init': {'num_cpus': None, (TaskRunnerV1 pid=2351) 'runtime_env': {'py_executable': 'uv -v run ' (TaskRunnerV1 pid=2351) '--frozen ' (TaskRunnerV1 pid=2351) '--all-packages ' (TaskRunnerV1 pid=2351) '--extra vllm ' (TaskRunnerV1 pid=2351) '--extra fsdp'}}, (TaskRunnerV1 pid=2351) 'timeline_json_file': None}, (TaskRunnerV1 pid=2351) 'reward': {'_target_': 'verl.workers.config.RewardConfig', (TaskRunnerV1 pid=2351) 'custom_reward_function': {'name': 'compute_score', (TaskRunnerV1 pid=2351) 'path': '/run/reward_tolerant.py'}, (TaskRunnerV1 pid=2351) 'num_workers': 8, (TaskRunnerV1 pid=2351) 'reward_manager': {'_target_': 'verl.workers.config.reward.RewardManagerConfig', (TaskRunnerV1 pid=2351) 'module': {'_target_': 'verl.trainer.config.config.ModuleConfig', (TaskRunnerV1 pid=2351) 'name': 'custom_reward_manager', (TaskRunnerV1 pid=2351) 'path': None}, (TaskRunnerV1 pid=2351) 'name': 'naive', (TaskRunnerV1 pid=2351) 'source': 'register'}, (TaskRunnerV1 pid=2351) 'reward_model': {'_target_': 'verl.workers.config.RewardModelConfig', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'enable_resource_pool': False, (TaskRunnerV1 pid=2351) 'model_path': None, (TaskRunnerV1 pid=2351) 'n_gpus_per_node': 8, (TaskRunnerV1 pid=2351) 'nnodes': 0, (TaskRunnerV1 pid=2351) 'rollout': {'_target_': 'verl.workers.config.RolloutConfig', (TaskRunnerV1 pid=2351) 'cudagraph_capture_sizes': None, (TaskRunnerV1 pid=2351) 'data_parallel_size': 1, (TaskRunnerV1 pid=2351) 'disable_log_stats': True, (TaskRunnerV1 pid=2351) 'disaggregation': {'bootstrap_port': None, (TaskRunnerV1 pid=2351) 'decode_replicas': 1, (TaskRunnerV1 pid=2351) 'decode_tensor_model_parallel_size': None, (TaskRunnerV1 pid=2351) 'enabled': False, (TaskRunnerV1 pid=2351) 'ib_device': None, (TaskRunnerV1 pid=2351) 'prefill_replicas': 1, (TaskRunnerV1 pid=2351) 'transfer_backend': 'nixl'}, (TaskRunnerV1 pid=2351) 'do_sample': True, (TaskRunnerV1 pid=2351) 'dtype': 'bfloat16', (TaskRunnerV1 pid=2351) 'enable_chunked_prefill': True, (TaskRunnerV1 pid=2351) 'enable_prefix_caching': True, (TaskRunnerV1 pid=2351) 'enforce_eager': True, (TaskRunnerV1 pid=2351) 'engine_kwargs': {}, (TaskRunnerV1 pid=2351) 'expert_parallel_size': 1, (TaskRunnerV1 pid=2351) 'free_cache_engine': True, (TaskRunnerV1 pid=2351) 'full_determinism': False, (TaskRunnerV1 pid=2351) 'gpu_memory_utilization': 0.5, (TaskRunnerV1 pid=2351) 'ignore_eos': False, (TaskRunnerV1 pid=2351) 'layered_summon': False, (TaskRunnerV1 pid=2351) 'limit_images': None, (TaskRunnerV1 pid=2351) 'load_format': 'auto', (TaskRunnerV1 pid=2351) 'max_model_len': None, (TaskRunnerV1 pid=2351) 'max_num_batched_tokens': 8192, (TaskRunnerV1 pid=2351) 'max_num_seqs': 1024, (TaskRunnerV1 pid=2351) 'mtp': None, (TaskRunnerV1 pid=2351) 'multi_stage_wake_up': False, (TaskRunnerV1 pid=2351) 'n': 1, (TaskRunnerV1 pid=2351) 'name': '???', (TaskRunnerV1 pid=2351) 'pipeline_model_parallel_size': 1, (TaskRunnerV1 pid=2351) 'prompt_length': 2048, (TaskRunnerV1 pid=2351) 'quantization': None, (TaskRunnerV1 pid=2351) 'quantization_config_file': None, (TaskRunnerV1 pid=2351) 'response_length': 2048, (TaskRunnerV1 pid=2351) 'scheduling_policy': 'fcfs', (TaskRunnerV1 pid=2351) 'seed': 42, (TaskRunnerV1 pid=2351) 'skip_tokenizer_init': False, (TaskRunnerV1 pid=2351) 'temperature': 1.0, (TaskRunnerV1 pid=2351) 'tensor_model_parallel_size': 2, (TaskRunnerV1 pid=2351) 'top_k': -1, (TaskRunnerV1 pid=2351) 'top_p': 1}}, (TaskRunnerV1 pid=2351) 'sandbox_fusion': {'_target_': 'verl.workers.config.SandboxFusionConfig', (TaskRunnerV1 pid=2351) 'max_concurrent': 64, (TaskRunnerV1 pid=2351) 'memory_limit_mb': 1024, (TaskRunnerV1 pid=2351) 'url': None}}, (TaskRunnerV1 pid=2351) 'reward_model': {'enable': None, (TaskRunnerV1 pid=2351) 'enable_resource_pool': None, (TaskRunnerV1 pid=2351) 'model': {'external_lib': None, (TaskRunnerV1 pid=2351) 'path': None, (TaskRunnerV1 pid=2351) 'trust_remote_code': None}, (TaskRunnerV1 pid=2351) 'n_gpus_per_node': None, (TaskRunnerV1 pid=2351) 'nnodes': None, (TaskRunnerV1 pid=2351) 'num_workers': None, (TaskRunnerV1 pid=2351) 'reward_loop_class_name': None, (TaskRunnerV1 pid=2351) 'reward_loop_module_path': None, (TaskRunnerV1 pid=2351) 'reward_loop_source': None, (TaskRunnerV1 pid=2351) 'reward_manager': None, (TaskRunnerV1 pid=2351) 'rollout': {'cudagraph_capture_sizes': None, (TaskRunnerV1 pid=2351) 'data_parallel_size': None, (TaskRunnerV1 pid=2351) 'disable_log_stats': None, (TaskRunnerV1 pid=2351) 'dtype': None, (TaskRunnerV1 pid=2351) 'enable_chunked_prefill': None, (TaskRunnerV1 pid=2351) 'enable_prefix_caching': None, (TaskRunnerV1 pid=2351) 'enforce_eager': None, (TaskRunnerV1 pid=2351) 'engine_kwargs': None, (TaskRunnerV1 pid=2351) 'expert_parallel_size': None, (TaskRunnerV1 pid=2351) 'free_cache_engine': None, (TaskRunnerV1 pid=2351) 'gpu_memory_utilization': None, (TaskRunnerV1 pid=2351) 'limit_images': None, (TaskRunnerV1 pid=2351) 'load_format': None, (TaskRunnerV1 pid=2351) 'max_model_len': None, (TaskRunnerV1 pid=2351) 'max_num_batched_tokens': None, (TaskRunnerV1 pid=2351) 'max_num_seqs': None, (TaskRunnerV1 pid=2351) 'name': None, (TaskRunnerV1 pid=2351) 'prompt_length': None, (TaskRunnerV1 pid=2351) 'response_length': None, (TaskRunnerV1 pid=2351) 'skip_tokenizer_init': None, (TaskRunnerV1 pid=2351) 'tensor_model_parallel_size': None}}, (TaskRunnerV1 pid=2351) 'sandbox_fusion': {'max_concurrent': None, (TaskRunnerV1 pid=2351) 'memory_limit_mb': None, (TaskRunnerV1 pid=2351) 'url': None},(raylet) INFO Waiting to acquire exclusive lock for `/workspace/verl` at `/tmp/uv-50f1055beec85b7b.lock` (raylet) DEBUG Found workspace root: `/workspace/verl` [repeated 9x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.) (raylet) DEBUG Adding root workspace member: `/workspace/verl` [repeated 9x across cluster] (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` [repeated 9x across cluster] (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` [repeated 9x across cluster] (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) [repeated 9x across cluster] (raylet) DEBUG Found project root: `/workspace/verl` [repeated 9x across cluster] (raylet) DEBUG No workspace root found, using project root [repeated 9x across cluster] (raylet) DEBUG Discovered project `verl` at: /workspace/verl [repeated 9x across cluster] (raylet) DEBUG No Python version file found in workspace: /workspace/verl [repeated 9x across cluster] (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata [repeated 9x across cluster] (raylet) DEBUG Checking for Python environment at: `.venv` [repeated 9x across cluster] (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` [repeated 9x across cluster] (raylet) DEBUG Skipping environment synchronization due to `--no-sync` [repeated 9x across cluster] (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 [repeated 9x across cluster] (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=41333 --object-store-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:41947 --session-name=session_2026-11-01_17-14-07_598925_196 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=3ae44e80d463b631d7fd94140af9fea5ec9e6078b42940dd11a74b17 --worker-id=94e270684d946afdb7db2b65e3769d1ad97492399f6b9f92a6e9ba51 --worker-launch-time-ms=1793553255747 --node-id=96099fbef72585e71fedb24085e672ce36048668f7de1351ec53c223 --runtime-env-hash=247405026` [repeated 9x across cluster] (raylet) DEBUG Spawned child 2602 in process group 1 [repeated 9x across cluster] (TaskRunnerV1 pid=2351) /workspace/verl/verl/trainer/ppo/v1/trainer_base.py:129: UserWarning: Disabled critic as algorithm.adv_estimator != gae. If it is not intended, please set critic.enable=True (TaskRunnerV1 pid=2351) self.use_critic = need_critic(self.config) (TaskRunnerV1 pid=2351) Setting TOKENIZERS_PARALLELISM=false for forked processes. (TaskRunnerV1 pid=2351) WARNING:2026-11-01 17:14:19,469:Setting TOKENIZERS_PARALLELISM=false for forked processes. (TaskRunnerV1 pid=2351) Filtering prompts longer than 128 tokens (num_proc=1): 0%| | 0/9000 [00:00=3.10, <3.13` from `requires-python` metadata (raylet) DEBUG Checking for Python environment at: `.venv` (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` (raylet) DEBUG Skipping environment synchronization due to `--no-sync` (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=41333 --object-store-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:41947 --session-name=session_2026-11-01_17-14-07_598925_196 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=3ae44e80d463b631d7fd94140af9fea5ec9e6078b42940dd11a74b17 --worker-id=fe30b11a4d6918085348792af924a72a7c488a6cec8f88aad4a647d0 --worker-launch-time-ms=1793553263179 --node-id=96099fbef72585e71fedb24085e672ce36048668f7de1351ec53c223 --runtime-env-hash=247405026` (raylet) DEBUG Spawned child 3179 in process group 1 (TaskRunnerV1 pid=2351) INFO:2026-11-01 17:14:23,643:create worker group dict_keys(['actor_rollout']) (WorkerDict pid=3270) /workspace/verl/verl/workers/engine/mindspeed/transformer_impl.py:24: UserWarning: NPU not support router replay for now. (WorkerDict pid=3270) from verl.utils.megatron.router_replay_patch import RouterReplay (WorkerDict pid=3270) Loading weights: 0%| | 0/311 [00:00=3.10, <3.13` from `requires-python` metadata (raylet) DEBUG Checking for Python environment at: `.venv` (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` (raylet) DEBUG Skipping environment synchronization due to `--no-sync` (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=41333 --object-store-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:41947 --session-name=session_2026-11-01_17-14-07_598925_196 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=3ae44e80d463b631d7fd94140af9fea5ec9e6078b42940dd11a74b17 --worker-id=7e1b011025810ee8a16f567fd2ee22dab3e1e693239a8d9904c1201b --worker-launch-time-ms=1793553263644 --node-id=96099fbef72585e71fedb24085e672ce36048668f7de1351ec53c223 --runtime-env-hash=757429697` (raylet) DEBUG Spawned child 3270 in process group 1 (WorkerDict pid=3270) Loading weights: 55%|█████▍ | 171/311 [00:00<00:00, 1688.91it/s] (WorkerDict pid=3270) Loading weights: 100%|██████████| 311/311 [00:00<00:00, 1752.97it/s] (TaskRunnerV1 pid=2351) INFO:2026-11-01 17:14:41,914:actor and ref model engine initialized (TaskRunnerV1 pid=2351) INFO:2026-11-01 17:14:41,967:reward loop manager initialized (raylet) DEBUG Found workspace root: `/workspace/verl` (raylet) DEBUG Adding root workspace member: `/workspace/verl` (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) (raylet) DEBUG Found project root: `/workspace/verl` (raylet) DEBUG No workspace root found, using project root (raylet) DEBUG Discovered project `verl` at: /workspace/verl (raylet) DEBUG No Python version file found in workspace: /workspace/verl (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata (raylet) DEBUG Checking for Python environment at: `.venv` (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` (raylet) DEBUG Skipping environment synchronization due to `--no-sync` (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=41333 --object-store-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:41947 --session-name=session_2026-11-01_17-14-07_598925_196 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=3ae44e80d463b631d7fd94140af9fea5ec9e6078b42940dd11a74b17 --worker-id=f14b0fb26e7cb39610a54fe7f448c3314d9bdaa0603b57d30f27741e --worker-launch-time-ms=1793553281933 --node-id=96099fbef72585e71fedb24085e672ce36048668f7de1351ec53c223 --runtime-env-hash=247405026` (raylet) DEBUG Spawned child 3430 in process group 1 (TaskRunnerV1 pid=2351) W1101 17:14:42.496000 2351 .venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py:140] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' (TaskRunnerV1 pid=2351) 'skip': {'_target_': 'verl.utils.skip.SkipManagerConfig', (TaskRunnerV1 pid=2351) 'async_rollout': {'action': 'cache', (TaskRunnerV1 pid=2351) 'dump_dir': '~/.verl/rollout_dump', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'steps': []}, (TaskRunnerV1 pid=2351) 'rollout': {'action': 'cache', (TaskRunnerV1 pid=2351) 'dump_dir': '~/.verl/rollout_dump', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'steps': []}, (TaskRunnerV1 pid=2351) 'rollout_tq': {'action': 'cache', (TaskRunnerV1 pid=2351) 'dump_dir': '~/.verl/rollout_dump', (TaskRunnerV1 pid=2351) 'enable': False, (TaskRunnerV1 pid=2351) 'steps': []}}, (TaskRunnerV1 pid=2351) 'trainer': {'balance_batch': True, (TaskRunnerV1 pid=2351) 'critic_warmup': 0, (TaskRunnerV1 pid=2351) 'default_hdfs_dir': None, (TaskRunnerV1 pid=2351) 'default_local_dir': '/checkpoints/char-count-qwen1.7b', (TaskRunnerV1 pid=2351) 'del_local_ckpt_after_load': False, (TaskRunnerV1 pid=2351) 'device': 'cuda', (TaskRunnerV1 pid=2351) 'esi_redundant_time': 0, (TaskRunnerV1 pid=2351) 'experiment_name': 'qwen3-1.7b-char-count-grpo-direct-1gpu', (TaskRunnerV1 pid=2351) 'log_val_generations': 0, (TaskRunnerV1 pid=2351) 'logger': ['console', 'mlflow'], (TaskRunnerV1 pid=2351) 'max_actor_ckpt_to_keep': None, (TaskRunnerV1 pid=2351) 'max_critic_ckpt_to_keep': None, (TaskRunnerV1 pid=2351) 'n_gpus_per_node': 1, (TaskRunnerV1 pid=2351) 'nnodes': 1, (TaskRunnerV1 pid=2351) 'project_name': 'verl_char_count', (TaskRunnerV1 pid=2351) 'ray_wait_register_center_timeout': 300, (TaskRunnerV1 pid=2351) 'resume_from_path': None, (TaskRunnerV1 pid=2351) 'resume_mode': 'auto', (TaskRunnerV1 pid=2351) 'rollout_data_dir': None, (TaskRunnerV1 pid=2351) 'save_freq': -1, (TaskRunnerV1 pid=2351) 'test_freq': 5, (TaskRunnerV1 pid=2351) 'total_epochs': 2, (TaskRunnerV1 pid=2351) 'total_training_steps': None, (TaskRunnerV1 pid=2351) 'use_v1': True, (TaskRunnerV1 pid=2351) 'v1': {'colocate_async': {'num_warmup_batches': 1}, (TaskRunnerV1 pid=2351) 'sampler': {'custom_sampler': {'name': None, 'path': None}, (TaskRunnerV1 pid=2351) 'max_off_policy_strategy': 'drop', (TaskRunnerV1 pid=2351) 'max_off_policy_threshold': 8, (TaskRunnerV1 pid=2351) 'sampler_kwargs': {}, (TaskRunnerV1 pid=2351) 'sync_refill_failed_groups': False}, (TaskRunnerV1 pid=2351) 'separate_async': {'num_warmup_batches': 1, (TaskRunnerV1 pid=2351) 'parameter_sync_step': 4}, (TaskRunnerV1 pid=2351) 'sync': {}, (TaskRunnerV1 pid=2351) 'trainer_mode': 'sync'}, (TaskRunnerV1 pid=2351) 'val_before_train': True, (TaskRunnerV1 pid=2351) 'val_only': False, (TaskRunnerV1 pid=2351) 'validation_data_dir': None}, (TaskRunnerV1 pid=2351) 'transfer_queue': {'backend': {'MooncakeStore': {'auto_init': False, (TaskRunnerV1 pid=2351) 'device_name': '', (TaskRunnerV1 pid=2351) 'global_segment_size': 4294967296, (TaskRunnerV1 pid=2351) 'local_buffer_size': 1073741824, (TaskRunnerV1 pid=2351) 'local_hostname': 'localhost', (TaskRunnerV1 pid=2351) 'master_server_address': 'localhost:50124', (TaskRunnerV1 pid=2351) 'metadata_server': 'localhost:50123', (TaskRunnerV1 pid=2351) 'protocol': 'tcp'}, (TaskRunnerV1 pid=2351) 'SimpleStorage': {'num_data_storage_units': 8, (TaskRunnerV1 pid=2351) 'total_storage_size': 100000}, (TaskRunnerV1 pid=2351) 'storage_backend': 'SimpleStorage'}, (TaskRunnerV1 pid=2351) 'enable': True, (TaskRunnerV1 pid=2351) 'metrics': {'enabled': False, 'port': 0}}} (TaskRunnerV1 pid=2351) Using dataset class: RLHFDataset (TaskRunnerV1 pid=2351) dataset len: 9000 (TaskRunnerV1 pid=2351) filter dataset len: 9000 (TaskRunnerV1 pid=2351) Using dataset class: RLHFDataset (TaskRunnerV1 pid=2351) dataset len: 1000 (TaskRunnerV1 pid=2351) filter dataset len: 1000 (TaskRunnerV1 pid=2351) colocated worker base class (WorkerDict pid=3270) Monkey patch state_dict in AutoModelForCausalLMWithValueHead. (WorkerDict pid=3270) Monkey patch _flash_attention_forward in transformers.integrations.flash_attention (WorkerDict pid=3270) Skipping monkey patch for Qwen3ForCausalLM as use_fused_kernels is False or fused_kernels_backend is torch (WorkerDict pid=3270) Qwen3ForCausalLM contains 1.72B parameters (WorkerDict pid=3270) Before FSDP, memory allocated (GB): 0.00, memory reserved (GB): 0.00, device memory used/total (GB): 0.55/47.58 (WorkerDict pid=3270) After FSDP, memory allocated (GB): 3.21, memory reserved (GB): 3.88, device memory used/total (GB): 4.94/47.58 (WorkerDict pid=3270) Total steps: 140, num_warmup_steps: 0 (vLLMHttpServer pid=3891) ['serve', (vLLMHttpServer pid=3891) '/models/Qwen3-1.7B', (vLLMHttpServer pid=3891) '--dtype', (vLLMHttpServer pid=3891) 'bfloat16', (vLLMHttpServer pid=3891) '--load_format', (vLLMHttpServer pid=3891) 'dummy', (vLLMHttpServer pid=3891) '--distributed_executor_backend', (vLLMHttpServer pid=3891) 'mp', (vLLMHttpServer pid=3891) '--worker_extension_cls', (vLLMHttpServer pid=3891) 'verl.workers.rollout.vllm_rollout.utils.vLLMColocateWorkerExtension', (vLLMHttpServer pid=3891) '--max_model_len', (vLLMHttpServer pid=3891) '40960', (vLLMHttpServer pid=3891) '--max_num_seqs', (vLLMHttpServer pid=3891) '1024', (vLLMHttpServer pid=3891) '--max_num_batched_tokens', (vLLMHttpServer pid=3891) '8192', (vLLMHttpServer pid=3891) '--enable_prefix_caching', (vLLMHttpServer pid=3891) '--enable_sleep_mode', (vLLMHttpServer pid=3891) '--logprobs_mode', (vLLMHttpServer pid=3891) 'processed_logprobs', (vLLMHttpServer pid=3891) '--enforce_eager', (vLLMHttpServer pid=3891) '--gpu_memory_utilization', (vLLMHttpServer pid=3891) '0.7', (vLLMHttpServer pid=3891) '--disable_log_stats', (vLLMHttpServer pid=3891) '--tensor_parallel_size', (vLLMHttpServer pid=3891) '1', (vLLMHttpServer pid=3891) '--seed', (vLLMHttpServer pid=3891) '42', (vLLMHttpServer pid=3891) '--override_generation_config',(TaskRunnerV1 pid=2351) INFO:2026-11-01 17:15:11,817:checkpoint engine manager initialized (raylet) DEBUG Found workspace root: `/workspace/verl` [repeated 8x across cluster] (raylet) DEBUG Adding root workspace member: `/workspace/verl` [repeated 8x across cluster] (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` [repeated 8x across cluster] (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` [repeated 8x across cluster] (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) [repeated 8x across cluster] (raylet) DEBUG Found project root: `/workspace/verl` [repeated 8x across cluster] (raylet) DEBUG No workspace root found, using project root [repeated 8x across cluster] (raylet) DEBUG Discovered project `verl` at: /workspace/verl [repeated 8x across cluster] (raylet) DEBUG No Python version file found in workspace: /workspace/verl [repeated 8x across cluster] (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata [repeated 8x across cluster] (raylet) DEBUG Checking for Python environment at: `.venv` [repeated 8x across cluster] (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` [repeated 8x across cluster] (raylet) DEBUG Skipping environment synchronization due to `--no-sync` [repeated 8x across cluster] (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 [repeated 8x across cluster] (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=41333 --object-store-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:41947 --session-name=session_2026-11-01_17-14-07_598925_196 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=3ae44e80d463b631d7fd94140af9fea5ec9e6078b42940dd11a74b17 --worker-id=a4c88ca2139e21492ec909b78346f83a81ab6b437949ab779eee07d8 --worker-launch-time-ms=1793553283742 --node-id=96099fbef72585e71fedb24085e672ce36048668f7de1351ec53c223 --runtime-env-hash=-1119239678` [repeated 8x across cluster] (raylet) DEBUG Spawned child 3891 in process group 1 [repeated 8x across cluster] (TaskRunnerV1 pid=2351) INFO:2026-11-01 17:15:11,970:Training from scratch (TaskRunnerV1 pid=2351) INFO:2026-11-01 17:15:11,970:all initialize finished, ready to fit (WorkerDict pid=3270) /workspace/verl/verl/plugin/platform/platform_cuda.py:99: FutureWarning: torch.cuda._set_allocator_settings is deprecated. Use torch._C._accelerator_setAllocatorSettings instead. (WorkerDict pid=3270) torch.cuda.memory._set_allocator_settings(settings) (TaskRunnerV1 pid=2351) INFO:2026-11-01 17:15:14,024:Using MLFlow tracking URI: http://:5000/mlflow (GlobalRequestLoadBalancer pid=4278) W1101 17:15:16.981000 4278 .venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py:140] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' (raylet) DEBUG Found workspace root: `/workspace/verl` [repeated 9x across cluster] (raylet) DEBUG Adding root workspace member: `/workspace/verl` [repeated 9x across cluster] (raylet) DEBUG Found workspace configuration at `/workspace/verl/pyproject.toml` [repeated 9x across cluster] (raylet) DEBUG Searching for user configuration in: `/root/.config/uv/uv.toml` [repeated 9x across cluster] (raylet) DEBUG uv 0.11.16 (x86_64-unknown-linux-musl) [repeated 9x across cluster] (raylet) DEBUG Found project root: `/workspace/verl` [repeated 9x across cluster] (raylet) DEBUG No workspace root found, using project root [repeated 9x across cluster] (raylet) DEBUG Discovered project `verl` at: /workspace/verl [repeated 9x across cluster] (raylet) DEBUG No Python version file found in workspace: /workspace/verl [repeated 9x across cluster] (raylet) DEBUG Using Python request `>=3.10, <3.13` from `requires-python` metadata [repeated 9x across cluster] (raylet) DEBUG Checking for Python environment at: `.venv` [repeated 9x across cluster] (raylet) DEBUG The project environment's Python version satisfies the request: `Python >=3.10, <3.13` [repeated 9x across cluster] (raylet) DEBUG Skipping environment synchronization due to `--no-sync` [repeated 9x across cluster] (raylet) DEBUG Using Python 3.12.3 interpreter at: /workspace/verl/.venv/bin/python3 [repeated 9x across cluster] (raylet) DEBUG Running `python /workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py --node-ip-address= --node-manager-port=41333 --object-store-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/plasma_store --raylet-name=/tmp/ray/session_2026-11-01_17-14-07_598925_196/sockets/raylet --redis-address=None --metrics-agent-port=0 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=0 --gcs-address=:41947 --session-name=session_2026-11-01_17-14-07_598925_196 --temp-dir=/tmp/ray --webui=:8265 --cluster-id=3ae44e80d463b631d7fd94140af9fea5ec9e6078b42940dd11a74b17 --worker-id=fb984514cd96b56a4a7d29704f07e6b30d2cf637c82a2a8bd2aa34d8 --worker-launch-time-ms=1793553313788 --node-id=96099fbef72585e71fedb24085e672ce36048668f7de1351ec53c223 --runtime-env-hash=247405026` [repeated 9x across cluster] (raylet) DEBUG Spawned child 4407 in process group 1 [repeated 9x across cluster] (pid=4391) WARNING:2026-11-01 17:15:18,089:Platform 'nvidia' (PlatformCUDA) is registered but not available. This may be due to this ray actor being a CPU-only actor. (pid=4391) /workspace/verl/verl/workers/engine/mindspeed/transformer_impl.py:24: UserWarning: NPU not support router replay for now. (pid=4391) from verl.utils.megatron.router_replay_patch import RouterReplay (AgentLoopWorkerTQ pid=4387) Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead. (AgentLoopWorkerTQ pid=4407) W1101 17:15:19.905000 4407 .venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py:140] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda' [repeated 8x across cluster] (RewardLoopWorker pid=3445) WARNING:2026-11-01 17:15:24,249:Platform 'nvidia' (PlatformCUDA) is registered but not available. This may be due to this ray actor being a CPU-only actor. [repeated 8x across cluster] (pid=4369) /workspace/verl/verl/workers/engine/mindspeed/transformer_impl.py:24: UserWarning: NPU not support router replay for now. [repeated 7x across cluster] (pid=4369) from verl.utils.megatron.router_replay_patch import RouterReplay [repeated 7x across cluster] (AgentLoopWorkerTQ pid=4407) Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead. [repeated 7x across cluster] (TaskRunnerV1 pid=2351) WARNING:2026-11-01 17:15:31,358:[MLflow] Metric key 'val-aux/char_count/reward/mean@1' sanitized to 'val-aux/char_count/reward/mean_at_1' due to invalid characters. (TaskRunnerV1 pid=2351) WARNING:2026-11-01 17:15:31,358:[MLflow] Metric key 'val-core/char_count/acc/mean@1' sanitized to 'val-core/char_count/acc/mean_at_1' due to invalid characters. (TaskRunnerV1 pid=2351) Training Progress: 0%| | 0/140 [00:00:35583'] (TaskRunnerV1 pid=2351) Checkpoint tracker file does not exist: /checkpoints/char-count-qwen1.7b/latest_checkpointed_iteration.txt (AgentLoopWorkerTQ pid=4387) Using dataset class: RLHFDataset (vLLMHttpServer pid=3891) WARNING 11-01 17:15:20 [input_processor.py:282] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat(). (vLLMHttpServer pid=3891) (Worker pid=4110) WARNING 11-01 17:15:20 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _topk_log_softmax_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. (TaskRunnerV1 pid=2351) ("Initial validation metrics: {'val-aux/char_count/reward/mean@1': " (TaskRunnerV1 pid=2351) "np.float64(0.213), 'val-core/char_count/acc/mean@1': np.float64(0.213), " (TaskRunnerV1 pid=2351) "'val-aux/num_turns/min': np.int64(2), 'val-aux/num_turns/max': np.int64(2), " (TaskRunnerV1 pid=2351) "'val-aux/num_turns/mean': np.float64(2.0)}") (TaskRunnerV1 pid=2351) step:0 - val-aux/char_count/reward/mean@1:np.float64(0.213) - val-core/char_count/acc/mean@1:np.float64(0.213) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) (AgentLoopWorkerTQ pid=4407) Using dataset class: RLHFDataset [repeated 7x across cluster] (TaskRunnerV1 pid=2351) step:1 - global_seqlen/min:162682.0 - global_seqlen/max:162682.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:162682.0 - global_seqlen/balanced_max:162682.0 - global_seqlen/mean:162682.0 - actor/entropy:0.19016145169734955 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6562013626098633 - training/rollout_probs_diff_mean:0.005369667429476976 - training/rollout_probs_diff_std:0.015494177117943764 - training/rollout_actor_probs_pearson_corr:0.9972605109214783 - rollout_corr/training_ppl:1.2132923603057861 - rollout_corr/training_log_ppl:0.19047920405864716 - rollout_corr/kl:0.0010534243192523718 - rollout_corr/k3_kl:0.0010999784572049975 - rollout_corr/rollout_ppl:1.2119578123092651 - rollout_corr/rollout_log_ppl:0.18941958248615265 - rollout_corr/log_ppl_diff:0.0010596244828775525 - rollout_corr/log_ppl_abs_diff:0.0034039129968732595 - rollout_corr/log_ppl_diff_max:0.0752248615026474 - rollout_corr/log_ppl_diff_min:-0.014120787382125854 - rollout_corr/ppl_ratio:1.0010724067687988 - rollout_corr/chi2_token:0.0021283626556396484 - rollout_corr/chi2_seq:0.3404674530029297 - actor/pg_clipfrac:0.0027214634077002606 - actor/ppo_kl:8.872978726988853e-05 - actor/pg_clipfrac_lower:7.5992459945458295e-06 - actor/pg_loss:0.0016323055315297097 - actor/loss:0.0016323060262948275 - actor/grad_norm:1.34375 - actor/perf/max_memory_allocated_gb:20.841578006744385 - actor/perf/max_memory_reserved_gb:27.505859375 - actor/perf/cpu_memory_used_gb:32.393914222717285 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:1 - training/epoch:0 - critic/score/mean:0.212890625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.212890625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.003027787897735834 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.003027787897735834 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:126.119140625 - response_length/max:128.0 - response_length/min:29.0 - response_length/clip_ratio:0.9228515625 - response_length_non_aborted/mean:126.119140625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:29.0 - response_length_non_aborted/clip_ratio:0.9228515625 - response/aborted_ratio:0.0 - prompt_length/mean:32.75 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.189695234003011 - timing_s/old_log_prob:10.05225909499859 - timing_s/adv:0.1883663099943078 - timing_s/update_actor:30.601878569999826 - timing_s/update_weights:1.536142431999906 - timing_s/step:50.86877752800501 - timing_per_token_ms/adv:0.0011578804661505748 - timing_per_token_ms/update_actor:0.18810857113878504 - timing_per_token_ms/gen:0.06341423841236284 - perf/total_num_tokens:162682 - perf/time_per_step:50.86877752800501 - perf/throughput:3198.071742739207 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:2 - global_seqlen/min:162586.0 - global_seqlen/max:162586.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:162586.0 - global_seqlen/balanced_max:162586.0 - global_seqlen/mean:162586.0 - actor/entropy:0.19407136738300323 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8868147730827332 - training/rollout_probs_diff_mean:0.005392906721681356 - training/rollout_probs_diff_std:0.015422482043504715 - training/rollout_actor_probs_pearson_corr:0.9974090456962585 - rollout_corr/training_ppl:1.2207915782928467 - rollout_corr/training_log_ppl:0.19642913341522217 - rollout_corr/kl:0.0010534541215747595 - rollout_corr/k3_kl:0.0009838908445090055 - rollout_corr/rollout_ppl:1.2194653749465942 - rollout_corr/rollout_log_ppl:0.1953776627779007 - rollout_corr/log_ppl_diff:0.0010514534078538418 - rollout_corr/log_ppl_abs_diff:0.0032287980429828167 - rollout_corr/log_ppl_diff_max:0.018895268440246582 - rollout_corr/log_ppl_diff_min:-0.012643679976463318 - rollout_corr/ppl_ratio:1.0010602474212646 - rollout_corr/chi2_token:0.0018444061279296875 - rollout_corr/chi2_seq:0.2478632926940918 - actor/pg_clipfrac:0.0024254355424394207 - actor/ppo_kl:7.954012289701502e-06 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.000680787445162423 - actor/loss:0.0006807880126871169 - actor/grad_norm:1.4765625 - actor/perf/max_memory_allocated_gb:20.85618829727173 - actor/perf/max_memory_reserved_gb:27.505859375 - actor/perf/cpu_memory_used_gb:41.45982646942139 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:2 - training/epoch:0 - critic/score/mean:0.203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.003698983695358038 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.003698983695358038 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:125.767578125 - response_length/max:128.0 - response_length/min:55.0 - response_length/clip_ratio:0.8955078125 - response_length_non_aborted/mean:125.767578125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:55.0 - response_length_non_aborted/clip_ratio:0.8955078125 - response/aborted_ratio:0.0 - prompt_length/mean:33.0078125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.176630410001962 - timing_s/old_log_prob:9.049291331000859 - timing_s/adv:0.18233055099699413 - timing_s/update_actor:28.192527016006352 - timing_s/update_weights:1.8485134079965064 - timing_s/step:47.62290369399852 - timing_per_token_ms/adv:0.0011214406590788514 - timing_per_token_ms/update_actor:0.17340070495618534 - timing_per_token_ms/gen:0.0634900564502505 - perf/total_num_tokens:162586 - perf/time_per_step:47.62290369399852 - perf/throughput:3414.0295401703784 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 2%|▏ | 3/140 [02:26<1:50:18, 48.31s/it] (TaskRunnerV1 pid=2351) Training Progress: 3%|▎ | 4/140 [03:13<1:48:59, 48.09s/it] (TransferQueueController pid=2438) 2026-11-01 17:19:33,218 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 25058, Total req/min: 4735.41, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.29 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=9454, req/min=1786.60, avg_time=0.000231s, max_time=0.035364s, min_time=0.000029s; SET_CUSTOM_META: req_count=9421, req/min=1780.36, avg_time=0.000009s, max_time=0.000265s, min_time=0.000003s; KV_LIST: req_count=32, req/min=6.05, avg_time=0.000157s, max_time=0.000370s, min_time=0.000029s; NOTIFY_DATA_UPDATE: req_count=6135, req/min=1159.38, avg_time=0.000074s, max_time=0.000846s, min_time=0.000030s; CLEAR_META: req_count=11, req/min=2.08, avg_time=0.001256s, max_time=0.001756s, min_time=0.000714s; KV_RETRIEVE_KEYS: req_count=5, req/min=0.94, avg_time=0.000142s, max_time=0.000207s, min_time=0.000100s (SimpleStorageUnit pid=2582) Per-operation statistics: PUT_DATA: req_count=780, req/min=146.41, avg_time=0.000614s, max_time=0.001696s, min_time=0.000321s; CLEAR_DATA: req_count=11, req/min=2.06, avg_time=0.001273s, max_time=0.003053s, min_time=0.000412s; GET_DATA: req_count=26, req/min=4.88, avg_time=0.000566s, max_time=0.001271s, min_time=0.000408s (TaskRunnerV1 pid=2351) Training Progress: 4%|▎ | 5/140 [04:11<1:55:46, 51.45s/it] (SimpleStorageUnit pid=2602) 2026-11-01 17:19:38,007 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_68b99b58: [Performance] Total success requests: 817, Total req/min: 152.94, Total avg process time: 0.0007s; [repeated 8x across cluster] (SimpleStorageUnit pid=2602) Time range: last 5.34 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2602) Per-operation statistics: PUT_DATA: req_count=780, req/min=146.01, avg_time=0.000688s, max_time=0.057237s, min_time=0.000305s; CLEAR_DATA: req_count=11, req/min=2.06, avg_time=0.001188s, max_time=0.003268s, min_time=0.000380s; GET_DATA: req_count=26, req/min=4.87, avg_time=0.000653s, max_time=0.001227s, min_time=0.000404s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) step:3 - global_seqlen/min:160548.0 - global_seqlen/max:160548.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160548.0 - global_seqlen/balanced_max:160548.0 - global_seqlen/mean:160548.0 - actor/entropy:0.19033439457416534 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9640275835990906 - training/rollout_probs_diff_mean:0.005377264693379402 - training/rollout_probs_diff_std:0.016377326101064682 - training/rollout_actor_probs_pearson_corr:0.9969335794448853 - rollout_corr/training_ppl:1.2185373306274414 - rollout_corr/training_log_ppl:0.19366934895515442 - rollout_corr/kl:0.0011943577555939555 - rollout_corr/k3_kl:0.0012575509026646614 - rollout_corr/rollout_ppl:1.2170000076293945 - rollout_corr/rollout_log_ppl:0.19246353209018707 - rollout_corr/log_ppl_diff:0.0012057996354997158 - rollout_corr/log_ppl_abs_diff:0.003432671306654811 - rollout_corr/log_ppl_diff_max:0.06202566623687744 - rollout_corr/log_ppl_diff_min:-0.023507609963417053 - rollout_corr/ppl_ratio:1.001218557357788 - rollout_corr/chi2_token:0.004505634307861328 - rollout_corr/chi2_seq:0.7161624431610107 - actor/pg_clipfrac:0.0027527950248137736 - actor/ppo_kl:-0.00010993572551853958 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0017944171268027276 - actor/loss:0.0017944148276001215 - actor/grad_norm:1.55859375 - actor/perf/max_memory_allocated_gb:20.85618829727173 - actor/perf/max_memory_reserved_gb:27.505859375 - actor/perf/cpu_memory_used_gb:41.4441180229187 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:3 - training/epoch:0 - critic/score/mean:0.2314453125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2314453125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008455902338027954 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008455902338027954 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:124.71484375 - response_length/max:128.0 - response_length/min:43.0 - response_length/clip_ratio:0.8564453125 - response_length_non_aborted/mean:124.71484375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:43.0 - response_length_non_aborted/clip_ratio:0.8564453125 - response/aborted_ratio:0.0 - prompt_length/mean:32.0703125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.193575428995246 - timing_s/old_log_prob:8.679078198998468 - timing_s/adv:0.3861835029965732 - timing_s/update_actor:27.94835448199592 - timing_s/update_weights:1.883228970997152 - timing_s/step:47.253229201996874 - timing_per_token_ms/adv:0.002405408370061123 - timing_per_token_ms/update_actor:0.17408098812813563 - timing_per_token_ms/gen:0.0641586700049742 - perf/total_num_tokens:160548 - perf/time_per_step:47.253229201996874 - perf/throughput:3397.609067386561 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:4 - global_seqlen/min:160949.0 - global_seqlen/max:160949.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160949.0 - global_seqlen/balanced_max:160949.0 - global_seqlen/mean:160949.0 - actor/entropy:0.1815498173236847 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8519467115402222 - training/rollout_probs_diff_mean:0.005264859646558762 - training/rollout_probs_diff_std:0.015664372593164444 - training/rollout_actor_probs_pearson_corr:0.9971005320549011 - rollout_corr/training_ppl:1.2062560319900513 - rollout_corr/training_log_ppl:0.1842433214187622 - rollout_corr/kl:0.0011996646644547582 - rollout_corr/k3_kl:0.0011326036183163524 - rollout_corr/rollout_ppl:1.2047616243362427 - rollout_corr/rollout_log_ppl:0.1830476075410843 - rollout_corr/log_ppl_diff:0.0011957106180489063 - rollout_corr/log_ppl_abs_diff:0.003458344377577305 - rollout_corr/log_ppl_diff_max:0.09722408652305603 - rollout_corr/log_ppl_diff_min:-0.014611005783081055 - rollout_corr/ppl_ratio:1.0012099742889404 - rollout_corr/chi2_token:0.0023784637451171875 - rollout_corr/chi2_seq:0.3027634620666504 - actor/pg_clipfrac:0.003381243591623691 - actor/ppo_kl:-8.642064850998093e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.004224109055940062 - actor/loss:0.0042241106275469065 - actor/grad_norm:1.607421875 - actor/perf/max_memory_allocated_gb:20.85618829727173 - actor/perf/max_memory_reserved_gb:27.505859375 - actor/perf/cpu_memory_used_gb:41.49174737930298 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:4 - training/epoch:0 - critic/score/mean:0.2763671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2763671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.010408451780676842 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.010408451780676842 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:125.2626953125 - response_length/max:128.0 - response_length/min:51.0 - response_length/clip_ratio:0.8671875 - response_length_non_aborted/mean:125.2626953125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:51.0 - response_length_non_aborted/clip_ratio:0.8671875 - response/aborted_ratio:0.0 - prompt_length/mean:31.9140625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.176499220004189 - timing_s/old_log_prob:8.74620885799959 - timing_s/adv:0.3805949489978957 - timing_s/update_actor:28.299588942005357 - timing_s/update_weights:1.8473347790059051 - timing_s/step:47.62276447899785 - timing_per_token_ms/adv:0.002364692846789329 - timing_per_token_ms/update_actor:0.17582954191703806 - timing_per_token_ms/gen:0.0637449361888234 - perf/total_num_tokens:160949 - perf/time_per_step:47.62276447899785 - perf/throughput:3379.665203412965 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:5 - global_seqlen/min:160472.0 - global_seqlen/max:160472.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160472.0 - global_seqlen/balanced_max:160472.0 - global_seqlen/mean:160472.0 - actor/entropy:0.1913892775774002 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8433346152305603 - training/rollout_probs_diff_mean:0.005519728176295757 - training/rollout_probs_diff_std:0.01587044633924961 - training/rollout_actor_probs_pearson_corr:0.9970671534538269 - rollout_corr/training_ppl:1.2183761596679688 - rollout_corr/training_log_ppl:0.19397476315498352 - rollout_corr/kl:0.0010648822644725442 - rollout_corr/k3_kl:0.0010299657005816698 - rollout_corr/rollout_ppl:1.2169885635375977 - rollout_corr/rollout_log_ppl:0.19287800788879395 - rollout_corr/log_ppl_diff:0.0010967759881168604 - rollout_corr/log_ppl_abs_diff:0.0033231182023882866 - rollout_corr/log_ppl_diff_max:0.03572145104408264 - rollout_corr/log_ppl_diff_min:-0.012361913919448853 - rollout_corr/ppl_ratio:1.0011067390441895 - rollout_corr/chi2_token:0.0019289255142211914 - rollout_corr/chi2_seq:0.24802911281585693 - actor/pg_clipfrac:0.0033048132124046488 - actor/ppo_kl:3.3269528216806066e-05 - actor/pg_clipfrac_lower:7.57025190978311e-06 - actor/pg_loss:0.0004028213443234563 - actor/loss:0.00040282122790813446 - actor/grad_norm:1.5341796875 - actor/perf/max_memory_allocated_gb:20.85618829727173 - actor/perf/max_memory_reserved_gb:27.505859375 - actor/perf/cpu_memory_used_gb:41.429969787597656 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.244) - val-core/char_count/acc/mean@1:np.float64(0.244) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:5 - training/epoch:0 - critic/score/mean:0.2470703125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2470703125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006094904150813818 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006094904150813818 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:124.921875 - response_length/max:128.0 - response_length/min:47.0 - response_length/clip_ratio:0.8564453125 - response_length_non_aborted/mean:124.921875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:47.0 - response_length_non_aborted/clip_ratio:0.8564453125 - response/aborted_ratio:0.0 - prompt_length/mean:31.7890625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.183361513998534 - timing_s/old_log_prob:8.871947319996252 - timing_s/adv:0.18386990900035016 - timing_s/update_actor:28.41819780300284 - timing_s/update_weights:1.8488202579974313 - timing_s/step:47.67315521300043 - timing_s/testing:9.623426705999009 - timing_per_token_ms/adv:0.0011458068011886819 - timing_per_token_ms/update_actor:0.17709131688396007 - timing_per_token_ms/gen:0.06397249463726183 - perf/total_num_tokens:160472 - perf/time_per_step:47.67315521300043 - perf/throughput:3366.0872514735383 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 4%|▍ | 6/140 [04:59<1:52:16, 50.27s/it] (TaskRunnerV1 pid=2351) Training Progress: 5%|▌ | 7/140 [05:46<1:49:28, 49.39s/it] (TaskRunnerV1 pid=2351) Training Progress: 6%|▌ | 8/140 [06:34<1:47:34, 48.90s/it] (TaskRunnerV1 pid=2351) step:6 - global_seqlen/min:161208.0 - global_seqlen/max:161208.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161208.0 - global_seqlen/balanced_max:161208.0 - global_seqlen/mean:161208.0 - actor/entropy:0.18598638474941254 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7417161464691162 - training/rollout_probs_diff_mean:0.005267527420073748 - training/rollout_probs_diff_std:0.015450491569936275 - training/rollout_actor_probs_pearson_corr:0.9972506165504456 - rollout_corr/training_ppl:1.2129590511322021 - rollout_corr/training_log_ppl:0.1896980255842209 - rollout_corr/kl:0.0009133520652540028 - rollout_corr/k3_kl:0.0010077658807858825 - rollout_corr/rollout_ppl:1.211837887763977 - rollout_corr/rollout_log_ppl:0.1887822300195694 - rollout_corr/log_ppl_diff:0.0009157916647382081 - rollout_corr/log_ppl_abs_diff:0.0031578014604747295 - rollout_corr/log_ppl_diff_max:0.01707857847213745 - rollout_corr/log_ppl_diff_min:-0.012610241770744324 - rollout_corr/ppl_ratio:1.0009241104125977 - rollout_corr/chi2_token:0.002305269241333008 - rollout_corr/chi2_seq:0.28059685230255127 - actor/pg_clipfrac:0.003416692547034472 - actor/ppo_kl:4.263868459020159e-05 - actor/pg_clipfrac_lower:8.812746576343974e-06 - actor/pg_loss:0.002965348328871187 - actor/loss:0.0029653478413820267 - actor/grad_norm:1.5244140625 - actor/perf/max_memory_allocated_gb:20.85618829727173 - actor/perf/max_memory_reserved_gb:27.505859375 - actor/perf/cpu_memory_used_gb:41.588075160980225 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:6 - training/epoch:0 - critic/score/mean:0.2607421875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2607421875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006897300016134977 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006897300016134977 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:125.65625 - response_length/max:128.0 - response_length/min:57.0 - response_length/clip_ratio:0.8818359375 - response_length_non_aborted/mean:125.65625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:57.0 - response_length_non_aborted/clip_ratio:0.8818359375 - response/aborted_ratio:0.0 - prompt_length/mean:31.7734375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.191134306005551 - timing_s/old_log_prob:8.800809802000003 - timing_s/adv:0.4626563959973282 - timing_s/update_actor:28.352965015998052 - timing_s/update_weights:1.8762183479993837 - timing_s/step:47.85884488799638 - timing_per_token_ms/adv:0.0028699344697367887 - timing_per_token_ms/update_actor:0.17587815130761533 - timing_per_token_ms/gen:0.06365902687457685 - perf/total_num_tokens:161208 - perf/time_per_step:47.85884488799638 - perf/throughput:3368.4055763835004 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:7 - global_seqlen/min:160143.0 - global_seqlen/max:160143.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160143.0 - global_seqlen/balanced_max:160143.0 - global_seqlen/mean:160143.0 - actor/entropy:0.18734198808670044 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5750328302383423 - training/rollout_probs_diff_mean:0.005331919062882662 - training/rollout_probs_diff_std:0.015512829646468163 - training/rollout_actor_probs_pearson_corr:0.9972415566444397 - rollout_corr/training_ppl:1.2102516889572144 - rollout_corr/training_log_ppl:0.18797945976257324 - rollout_corr/kl:0.0009947316721081734 - rollout_corr/k3_kl:0.001073885359801352 - rollout_corr/rollout_ppl:1.2090309858322144 - rollout_corr/rollout_log_ppl:0.186996191740036 - rollout_corr/log_ppl_diff:0.0009832687210291624 - rollout_corr/log_ppl_abs_diff:0.003389733610674739 - rollout_corr/log_ppl_diff_max:0.03666144609451294 - rollout_corr/log_ppl_diff_min:-0.014476671814918518 - rollout_corr/ppl_ratio:1.0009942054748535 - rollout_corr/chi2_token:0.0022467374801635742 - rollout_corr/chi2_seq:0.33406221866607666 - actor/pg_clipfrac:0.003305543985334225 - actor/ppo_kl:1.6239974759931413e-05 - actor/pg_clipfrac_lower:7.581271347589791e-06 - actor/pg_loss:-0.00028228033625055104 - actor/loss:-0.00028227921575307846 - actor/grad_norm:1.54248046875 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.58394432067871 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:7 - training/epoch:0 - critic/score/mean:0.3037109375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3037109375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008051424287259579 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008051424287259579 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:124.0458984375 - response_length/max:128.0 - response_length/min:41.0 - response_length/clip_ratio:0.822265625 - response_length_non_aborted/mean:124.0458984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:41.0 - response_length_non_aborted/clip_ratio:0.822265625 - response/aborted_ratio:0.0 - prompt_length/mean:32.34375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.181870517997595 - timing_s/old_log_prob:8.751493584000855 - timing_s/adv:0.40622977900056867 - timing_s/update_actor:28.097784334000607 - timing_s/update_weights:1.8472542629970121 - timing_s/step:47.448654569998325 - timing_per_token_ms/adv:0.002536668970860847 - timing_per_token_ms/update_actor:0.17545433977133318 - timing_per_token_ms/gen:0.06441251204898006 - perf/total_num_tokens:160143 - perf/time_per_step:47.448654569998325 - perf/throughput:3375.0798932296398 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:8 - global_seqlen/min:161625.0 - global_seqlen/max:161625.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161625.0 - global_seqlen/balanced_max:161625.0 - global_seqlen/mean:161625.0 - actor/entropy:0.19171635806560516 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8041819334030151 - training/rollout_probs_diff_mean:0.005365625023841858 - training/rollout_probs_diff_std:0.015400360338389874 - training/rollout_actor_probs_pearson_corr:0.9973819255828857 - rollout_corr/training_ppl:1.2190439701080322 - rollout_corr/training_log_ppl:0.1946794092655182 - rollout_corr/kl:0.0012301767710596323 - rollout_corr/k3_kl:0.001056150533258915 - rollout_corr/rollout_ppl:1.2174787521362305 - rollout_corr/rollout_log_ppl:0.19345130026340485 - rollout_corr/log_ppl_diff:0.0012280954979360104 - rollout_corr/log_ppl_abs_diff:0.003482606029137969 - rollout_corr/log_ppl_diff_max:0.055151358246803284 - rollout_corr/log_ppl_diff_min:-0.0125579833984375 - rollout_corr/ppl_ratio:1.0012396574020386 - rollout_corr/chi2_token:0.0016961097717285156 - rollout_corr/chi2_seq:0.2741185426712036 - actor/pg_clipfrac:0.0031703088170615956 - actor/ppo_kl:6.224651906450163e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.003464980451099109 - actor/loss:0.0034649809822440147 - actor/grad_norm:1.5859375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.660133361816406 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:8 - training/epoch:0 - critic/score/mean:0.2548828125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2548828125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007506677880883217 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007506677880883217 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:125.3447265625 - response_length/max:128.0 - response_length/min:61.0 - response_length/clip_ratio:0.8798828125 - response_length_non_aborted/mean:125.3447265625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:61.0 - response_length_non_aborted/clip_ratio:0.8798828125 - response/aborted_ratio:0.0 - prompt_length/mean:32.4921875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.182603829998698 - timing_s/old_log_prob:8.76995687699673 - timing_s/adv:0.2643824010010576 - timing_s/update_actor:28.45525724699837 - timing_s/update_weights:1.8782793449936435 - timing_s/step:47.71409078600118 - timing_per_token_ms/adv:0.0016357766496585156 - timing_per_token_ms/update_actor:0.1760572760835166 - timing_per_token_ms/gen:0.06375077972465543 - perf/total_num_tokens:161625 - perf/time_per_step:47.71409078600118 - perf/throughput:3387.3641378789325 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 6%|▋ | 9/140 [07:22<1:46:02, 48.57s/it] (TaskRunnerV1 pid=2351) Training Progress: 7%|▋ | 10/140 [08:19<1:50:51, 51.16s/it] (TransferQueueController pid=2438) 2026-11-01 17:24:38,379 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 35680, Total req/min: 7015.31, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.09 minutes; (TransferQueueController pid=2438) Per-operation statistics: SET_CUSTOM_META: req_count=13706, req/min=2694.84, avg_time=0.000009s, max_time=0.000230s, min_time=0.000003s; KV_RETRIEVE_META: req_count=13750, req/min=2703.49, avg_time=0.000226s, max_time=0.065762s, min_time=0.000063s; KV_LIST: req_count=40, req/min=7.86, avg_time=0.000258s, max_time=0.001019s, min_time=0.000042s; NOTIFY_DATA_UPDATE: req_count=8162, req/min=1604.79, avg_time=0.000072s, max_time=0.001035s, min_time=0.000030s; CLEAR_META: req_count=16, req/min=3.15, avg_time=0.001591s, max_time=0.003093s, min_time=0.000770s; KV_RETRIEVE_KEYS: req_count=6, req/min=1.18, avg_time=0.000108s, max_time=0.000121s, min_time=0.000098s (SimpleStorageUnit pid=2599) Per-operation statistics: PUT_DATA: req_count=1038, req/min=206.66, avg_time=0.000584s, max_time=0.001963s, min_time=0.000325s; CLEAR_DATA: req_count=16, req/min=3.19, avg_time=0.001371s, max_time=0.003341s, min_time=0.000417s; GET_DATA: req_count=35, req/min=6.97, avg_time=0.000657s, max_time=0.001265s, min_time=0.000387s (TaskRunnerV1 pid=2351) Training Progress: 8%|▊ | 11/140 [09:07<1:47:46, 50.13s/it] (TaskRunnerV1 pid=2351) step:9 - global_seqlen/min:161694.0 - global_seqlen/max:161694.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161694.0 - global_seqlen/balanced_max:161694.0 - global_seqlen/mean:161694.0 - actor/entropy:0.18305161595344543 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7399729490280151 - training/rollout_probs_diff_mean:0.005189181305468082 - training/rollout_probs_diff_std:0.015749990940093994 - training/rollout_actor_probs_pearson_corr:0.9971847534179688 - rollout_corr/training_ppl:1.2075095176696777 - rollout_corr/training_log_ppl:0.1851092427968979 - rollout_corr/kl:0.0009948391234502196 - rollout_corr/k3_kl:0.0010183728300035 - rollout_corr/rollout_ppl:1.206288456916809 - rollout_corr/rollout_log_ppl:0.18412382900714874 - rollout_corr/log_ppl_diff:0.0009853977244347334 - rollout_corr/log_ppl_abs_diff:0.0033205277286469936 - rollout_corr/log_ppl_diff_max:0.029453717172145844 - rollout_corr/log_ppl_diff_min:-0.017622798681259155 - rollout_corr/ppl_ratio:1.00099515914917 - rollout_corr/chi2_token:0.002012968063354492 - rollout_corr/chi2_seq:0.3750736713409424 - actor/pg_clipfrac:0.0027216664724013144 - actor/ppo_kl:0.0001703357321882019 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0003628034610301256 - actor/loss:-0.000362802529707551 - actor/grad_norm:1.39453125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.73910903930664 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:9 - training/epoch:0 - critic/score/mean:0.212890625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.212890625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004987942986190319 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004987942986190319 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:125.365234375 - response_length/max:128.0 - response_length/min:51.0 - response_length/clip_ratio:0.8759765625 - response_length_non_aborted/mean:125.365234375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:51.0 - response_length_non_aborted/clip_ratio:0.8759765625 - response/aborted_ratio:0.0 - prompt_length/mean:32.5390625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.183214518001478 - timing_s/old_log_prob:8.82188056599989 - timing_s/adv:0.18315871300001163 - timing_s/update_actor:28.337414804998843 - timing_s/update_weights:1.8388382909979555 - timing_s/step:47.53922263700224 - timing_per_token_ms/adv:0.0011327489764617837 - timing_per_token_ms/update_actor:0.1752533477123384 - timing_per_token_ms/gen:0.06374510818391167 - perf/total_num_tokens:161694 - perf/time_per_step:47.53922263700224 - perf/throughput:3401.2756421083163 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:10 - global_seqlen/min:161121.0 - global_seqlen/max:161121.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161121.0 - global_seqlen/balanced_max:161121.0 - global_seqlen/mean:161121.0 - actor/entropy:0.17471946775913239 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.857214093208313 - training/rollout_probs_diff_mean:0.0051029580645263195 - training/rollout_probs_diff_std:0.01563258282840252 - training/rollout_actor_probs_pearson_corr:0.9971280097961426 - rollout_corr/training_ppl:1.1987699270248413 - rollout_corr/training_log_ppl:0.1783674955368042 - rollout_corr/kl:0.0010880863992497325 - rollout_corr/k3_kl:0.0009706819546408951 - rollout_corr/rollout_ppl:1.1974310874938965 - rollout_corr/rollout_log_ppl:0.17727461457252502 - rollout_corr/log_ppl_diff:0.0010928540723398328 - rollout_corr/log_ppl_abs_diff:0.0030854353681206703 - rollout_corr/log_ppl_diff_max:0.035386137664318085 - rollout_corr/log_ppl_diff_min:-0.010301679372787476 - rollout_corr/ppl_ratio:1.0011016130447388 - rollout_corr/chi2_token:0.0015668869018554688 - rollout_corr/chi2_seq:0.15865683555603027 - actor/pg_clipfrac:0.0033788871320818243 - actor/ppo_kl:0.00030394478108064504 - actor/pg_clipfrac_lower:7.57025190978311e-06 - actor/pg_loss:0.000306593079585582 - actor/loss:0.00030659325420856476 - actor/grad_norm:1.521484375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.70225811004639 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.277) - val-core/char_count/acc/mean@1:np.float64(0.277) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:10 - training/epoch:0 - critic/score/mean:0.2958984375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2958984375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008767735213041306 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008767735213041306 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:124.4619140625 - response_length/max:128.0 - response_length/min:50.0 - response_length/clip_ratio:0.8330078125 - response_length_non_aborted/mean:124.4619140625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:50.0 - response_length_non_aborted/clip_ratio:0.8330078125 - response/aborted_ratio:0.0 - prompt_length/mean:32.8828125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.182880806998583 - timing_s/old_log_prob:8.827753760000633 - timing_s/adv:0.187872315000277 - timing_s/update_actor:28.71084271800646 - timing_s/update_weights:1.8526819499966223 - timing_s/step:47.93798118500126 - timing_s/testing:8.911990681001043 - timing_per_token_ms/adv:0.0011660324538717922 - timing_per_token_ms/update_actor:0.1781942932206631 - timing_per_token_ms/gen:0.06420513936553902 - perf/total_num_tokens:161121 - perf/time_per_step:47.93798118500126 - perf/throughput:3361.0301480616213 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:11 - global_seqlen/min:159996.0 - global_seqlen/max:159996.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159996.0 - global_seqlen/balanced_max:159996.0 - global_seqlen/mean:159996.0 - actor/entropy:0.16904009878635406 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9699772000312805 - training/rollout_probs_diff_mean:0.005029493942856789 - training/rollout_probs_diff_std:0.015331610105931759 - training/rollout_actor_probs_pearson_corr:0.9970828294754028 - rollout_corr/training_ppl:1.1881147623062134 - rollout_corr/training_log_ppl:0.1694178730249405 - rollout_corr/kl:0.0011457358486950397 - rollout_corr/k3_kl:0.00104486383497715 - rollout_corr/rollout_ppl:1.1867026090621948 - rollout_corr/rollout_log_ppl:0.16827544569969177 - rollout_corr/log_ppl_diff:0.0011424255790188909 - rollout_corr/log_ppl_abs_diff:0.0033829910680651665 - rollout_corr/log_ppl_diff_max:0.059229254722595215 - rollout_corr/log_ppl_diff_min:-0.012526661157608032 - rollout_corr/ppl_ratio:1.0011547803878784 - rollout_corr/chi2_token:0.001698136329650879 - rollout_corr/chi2_seq:0.2601628303527832 - actor/pg_clipfrac:0.003040390021245306 - actor/ppo_kl:-0.0001269094027520623 - actor/pg_clipfrac_lower:7.58265074788748e-06 - actor/pg_loss:0.0025155666517093778 - actor/loss:0.0025155670009553432 - actor/grad_norm:1.4375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.72940015792847 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:11 - training/epoch:0 - critic/score/mean:0.3154296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3154296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.009169697761535645 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.009169697761535645 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:124.26953125 - response_length/max:128.0 - response_length/min:46.0 - response_length/clip_ratio:0.822265625 - response_length_non_aborted/mean:124.26953125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:46.0 - response_length_non_aborted/clip_ratio:0.822265625 - response/aborted_ratio:0.0 - prompt_length/mean:31.9765625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.195281794003677 - timing_s/old_log_prob:8.825198177000857 - timing_s/adv:0.18217588499828707 - timing_s/update_actor:28.262622952999664 - timing_s/update_weights:1.8367557150049834 - timing_s/step:47.47007562600629 - timing_per_token_ms/adv:0.0011386277469329676 - timing_per_token_ms/update_actor:0.17664580960148793 - timing_per_token_ms/gen:0.06440198813381068 - perf/total_num_tokens:159996 - perf/time_per_step:47.47007562600629 - perf/throughput:3370.4601876038896 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 9%|▊ | 12/140 [09:55<1:45:31, 49.46s/it] (SimpleStorageUnit pid=2583) 2026-11-01 17:24:38,567 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1084, Total req/min: 216.07, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.02 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: PUT_DATA: req_count=1033, req/min=205.90, avg_time=0.000626s, max_time=0.003642s, min_time=0.000324s; CLEAR_DATA: req_count=16, req/min=3.19, avg_time=0.001655s, max_time=0.004074s, min_time=0.000416s; GET_DATA: req_count=35, req/min=6.98, avg_time=0.000716s, max_time=0.001268s, min_time=0.000430s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 9%|▉ | 13/140 [10:43<1:43:37, 48.96s/it] (TaskRunnerV1 pid=2351) Training Progress: 10%|█ | 14/140 [11:30<1:41:45, 48.46s/it] (TaskRunnerV1 pid=2351) step:12 - global_seqlen/min:161940.0 - global_seqlen/max:161940.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:161940.0 - global_seqlen/balanced_max:161940.0 - global_seqlen/mean:161940.0 - actor/entropy:0.1717355102300644 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6044729948043823 - training/rollout_probs_diff_mean:0.0050427233800292015 - training/rollout_probs_diff_std:0.01529897004365921 - training/rollout_actor_probs_pearson_corr:0.9971686005592346 - rollout_corr/training_ppl:1.1935926675796509 - rollout_corr/training_log_ppl:0.17369259893894196 - rollout_corr/kl:0.0010416379664093256 - rollout_corr/k3_kl:0.0009642231743782759 - rollout_corr/rollout_ppl:1.1922930479049683 - rollout_corr/rollout_log_ppl:0.1726306676864624 - rollout_corr/log_ppl_diff:0.001061920658685267 - rollout_corr/log_ppl_abs_diff:0.0031536025926470757 - rollout_corr/log_ppl_diff_max:0.02652997523546219 - rollout_corr/log_ppl_diff_min:-0.010349243879318237 - rollout_corr/ppl_ratio:1.0010703802108765 - rollout_corr/chi2_token:0.0017467737197875977 - rollout_corr/chi2_seq:0.22174763679504395 - actor/pg_clipfrac:0.003028410336507174 - actor/ppo_kl:4.6786654669025055e-05 - actor/pg_clipfrac_lower:7.584031360844771e-06 - actor/pg_loss:0.0025987939443439245 - actor/loss:0.0025987927801907063 - actor/grad_norm:1.5771484375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.863534450531006 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:12 - training/epoch:0 - critic/score/mean:0.2578125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2578125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007435525767505169 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007435525767505169 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:125.56640625 - response_length/max:128.0 - response_length/min:52.0 - response_length/clip_ratio:0.8779296875 - response_length_non_aborted/mean:125.56640625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:52.0 - response_length_non_aborted/clip_ratio:0.8779296875 - response/aborted_ratio:0.0 - prompt_length/mean:32.578125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.185407956996642 - timing_s/old_log_prob:8.944910578997224 - timing_s/adv:0.18503984100243542 - timing_s/update_actor:28.47352949600463 - timing_s/update_weights:1.8522513810021337 - timing_s/step:47.807993927002826 - timing_per_token_ms/adv:0.0011426444424011079 - timing_per_token_ms/update_actor:0.17582764910463525 - timing_per_token_ms/gen:0.06366004010730006 - perf/total_num_tokens:161940 - perf/time_per_step:47.807993927002826 - perf/throughput:3387.2996270720605 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:13 - global_seqlen/min:160032.0 - global_seqlen/max:160032.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160032.0 - global_seqlen/balanced_max:160032.0 - global_seqlen/mean:160032.0 - actor/entropy:0.17951498925685883 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9626563787460327 - training/rollout_probs_diff_mean:0.0053293295204639435 - training/rollout_probs_diff_std:0.016459312289953232 - training/rollout_actor_probs_pearson_corr:0.9968244433403015 - rollout_corr/training_ppl:1.2024893760681152 - rollout_corr/training_log_ppl:0.18124014139175415 - rollout_corr/kl:0.001125713693909347 - rollout_corr/k3_kl:0.0012032454833388329 - rollout_corr/rollout_ppl:1.2010633945465088 - rollout_corr/rollout_log_ppl:0.18010391294956207 - rollout_corr/log_ppl_diff:0.0011362191289663315 - rollout_corr/log_ppl_abs_diff:0.0033828946761786938 - rollout_corr/log_ppl_diff_max:0.06842675805091858 - rollout_corr/log_ppl_diff_min:-0.012195289134979248 - rollout_corr/ppl_ratio:1.0011519193649292 - rollout_corr/chi2_token:0.0021486282348632812 - rollout_corr/chi2_seq:0.2563912868499756 - actor/pg_clipfrac:0.0030377459673521416 - actor/ppo_kl:0.00027119323037065135 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.00048635767598170787 - actor/loss:-0.0004863583017140627 - actor/grad_norm:1.5048828125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.78348255157471 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:13 - training/epoch:0 - critic/score/mean:0.283203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.283203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.012721889652311802 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.012721889652311802 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:123.5390625 - response_length/max:128.0 - response_length/min:46.0 - response_length/clip_ratio:0.8212890625 - response_length_non_aborted/mean:123.5390625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:46.0 - response_length_non_aborted/clip_ratio:0.8212890625 - response/aborted_ratio:0.0 - prompt_length/mean:32.7421875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.185515463999764 - timing_s/old_log_prob:8.862409730994841 - timing_s/adv:0.22753145600290736 - timing_s/update_actor:28.200561839999864 - timing_s/update_weights:1.845121783997456 - timing_s/step:47.662826769003004 - timing_per_token_ms/adv:0.0014217872425696572 - timing_per_token_ms/update_actor:0.17621826784642985 - timing_per_token_ms/gen:0.0647055861000424 - perf/total_num_tokens:160032 - perf/time_per_step:47.662826769003004 - perf/throughput:3357.5851632886165 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:14 - global_seqlen/min:159777.0 - global_seqlen/max:159777.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159777.0 - global_seqlen/balanced_max:159777.0 - global_seqlen/mean:159777.0 - actor/entropy:0.16387896239757538 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6355934739112854 - training/rollout_probs_diff_mean:0.004958816338330507 - training/rollout_probs_diff_std:0.015737321227788925 - training/rollout_actor_probs_pearson_corr:0.9968262910842896 - rollout_corr/training_ppl:1.1849409341812134 - rollout_corr/training_log_ppl:0.1666695773601532 - rollout_corr/kl:0.001036725821904838 - rollout_corr/k3_kl:0.0010162661783397198 - rollout_corr/rollout_ppl:1.1836925745010376 - rollout_corr/rollout_log_ppl:0.1656673550605774 - rollout_corr/log_ppl_diff:0.0010022249771282077 - rollout_corr/log_ppl_abs_diff:0.003232449758797884 - rollout_corr/log_ppl_diff_max:0.034833185374736786 - rollout_corr/log_ppl_diff_min:-0.01775836944580078 - rollout_corr/ppl_ratio:1.0010119676589966 - rollout_corr/chi2_token:0.002161860466003418 - rollout_corr/chi2_seq:0.2668989896774292 - actor/pg_clipfrac:0.003021437301261661 - actor/ppo_kl:8.070598672323584e-06 - actor/pg_clipfrac_lower:1.5264926939077363e-05 - actor/pg_loss:-0.0007677273824810982 - actor/loss:-0.0007677273824810982 - actor/grad_norm:1.517578125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.7784857749939 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:14 - training/epoch:0 - critic/score/mean:0.2998046875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2998046875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004307831637561321 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004307831637561321 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:123.7119140625 - response_length/max:128.0 - response_length/min:43.0 - response_length/clip_ratio:0.8056640625 - response_length_non_aborted/mean:123.7119140625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:43.0 - response_length_non_aborted/clip_ratio:0.8056640625 - response/aborted_ratio:0.0 - prompt_length/mean:32.3203125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.182836388004944 - timing_s/old_log_prob:9.019450002000667 - timing_s/adv:0.18084328300028574 - timing_s/update_actor:27.77941201700014 - timing_s/update_weights:1.8540760680043604 - timing_s/step:47.183131270998274 - timing_per_token_ms/adv:0.001131848031946311 - timing_per_token_ms/update_actor:0.17386364756504466 - timing_per_token_ms/gen:0.0645940305807891 - perf/total_num_tokens:159777 - perf/time_per_step:47.183131270998274 - perf/throughput:3386.3161620689007 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 11%|█ | 15/140 [12:26<1:45:57, 50.86s/it] (TaskRunnerV1 pid=2351) Training Progress: 11%|█▏ | 16/140 [13:13<1:42:35, 49.64s/it] (TaskRunnerV1 pid=2351) Training Progress: 12%|█▏ | 17/140 [14:00<1:40:11, 48.88s/it] (TaskRunnerV1 pid=2351) step:15 - global_seqlen/min:160772.0 - global_seqlen/max:160772.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:160772.0 - global_seqlen/balanced_max:160772.0 - global_seqlen/mean:160772.0 - actor/entropy:0.16596263647079468 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9332179427146912 - training/rollout_probs_diff_mean:0.004944054875522852 - training/rollout_probs_diff_std:0.016097014769911766 - training/rollout_actor_probs_pearson_corr:0.9968441128730774 - rollout_corr/training_ppl:1.1886405944824219 - rollout_corr/training_log_ppl:0.1695200502872467 - rollout_corr/kl:0.0011682292679324746 - rollout_corr/k3_kl:0.0010689441114664078 - rollout_corr/rollout_ppl:1.1871917247772217 - rollout_corr/rollout_log_ppl:0.16837167739868164 - rollout_corr/log_ppl_diff:0.0011483649723231792 - rollout_corr/log_ppl_abs_diff:0.0034213417675346136 - rollout_corr/log_ppl_diff_max:0.04535245895385742 - rollout_corr/log_ppl_diff_min:-0.013000428676605225 - rollout_corr/ppl_ratio:1.001159906387329 - rollout_corr/chi2_token:0.001735091209411621 - rollout_corr/chi2_seq:0.2763187885284424 - actor/pg_clipfrac:0.0024089595487263677 - actor/ppo_kl:5.120826350927625e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0007183760753832757 - actor/loss:-0.0007183768320828676 - actor/grad_norm:1.55029296875 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.881454944610596 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.302) - val-core/char_count/acc/mean@1:np.float64(0.302) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:15 - training/epoch:0 - critic/score/mean:0.228515625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.228515625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0073106796480715275 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0073106796480715275 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:123.92578125 - response_length/max:128.0 - response_length/min:54.0 - response_length/clip_ratio:0.833984375 - response_length_non_aborted/mean:123.92578125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:54.0 - response_length_non_aborted/clip_ratio:0.833984375 - response/aborted_ratio:0.0 - prompt_length/mean:33.078125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.19964959799836 - timing_s/old_log_prob:8.69713260300341 - timing_s/adv:0.3852143559997785 - timing_s/update_actor:28.056353336003667 - timing_s/update_weights:1.8366591700032586 - timing_s/step:47.33864924299996 - timing_s/testing:8.738803421001649 - timing_per_token_ms/adv:0.002396028885625473 - timing_per_token_ms/update_actor:0.17451019665118098 - timing_per_token_ms/gen:0.06461504805357257 - perf/total_num_tokens:160772 - perf/time_per_step:47.33864924299996 - perf/throughput:3396.2101278961522 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:16 - global_seqlen/min:158349.0 - global_seqlen/max:158349.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158349.0 - global_seqlen/balanced_max:158349.0 - global_seqlen/mean:158349.0 - actor/entropy:0.17546138167381287 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9004998207092285 - training/rollout_probs_diff_mean:0.005166484508663416 - training/rollout_probs_diff_std:0.01513142790645361 - training/rollout_actor_probs_pearson_corr:0.9972429275512695 - rollout_corr/training_ppl:1.200566291809082 - rollout_corr/training_log_ppl:0.17937976121902466 - rollout_corr/kl:0.0008145684842020273 - rollout_corr/k3_kl:0.0009434752864763141 - rollout_corr/rollout_ppl:1.199455976486206 - rollout_corr/rollout_log_ppl:0.17852088809013367 - rollout_corr/log_ppl_diff:0.0008588608470745385 - rollout_corr/log_ppl_abs_diff:0.00323740323074162 - rollout_corr/log_ppl_diff_max:0.04463827610015869 - rollout_corr/log_ppl_diff_min:-0.013690143823623657 - rollout_corr/ppl_ratio:1.0008682012557983 - rollout_corr/chi2_token:0.002134561538696289 - rollout_corr/chi2_seq:0.30814671516418457 - actor/pg_clipfrac:0.0032235012768069282 - actor/ppo_kl:3.6438622487366956e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.002489833364961669 - actor/loss:0.0024898326955735683 - actor/grad_norm:1.58203125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.94918918609619 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:16 - training/epoch:0 - critic/score/mean:0.2998046875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2998046875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.009101747535169125 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.009101747535169125 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:122.5205078125 - response_length/max:128.0 - response_length/min:35.0 - response_length/clip_ratio:0.7705078125 - response_length_non_aborted/mean:122.5205078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:35.0 - response_length_non_aborted/clip_ratio:0.7705078125 - response/aborted_ratio:0.0 - prompt_length/mean:32.1171875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.217061262002971 - timing_s/old_log_prob:8.65919287900033 - timing_s/adv:0.18757911199645605 - timing_s/update_actor:27.586660184999346 - timing_s/update_weights:1.8557963599960203 - timing_s/step:46.67592279499513 - timing_per_token_ms/adv:0.0011845929686733485 - timing_per_token_ms/update_actor:0.17421429996399942 - timing_per_token_ms/gen:0.0654949447398233 - perf/total_num_tokens:158349 - perf/time_per_step:46.67592279499513 - perf/throughput:3392.519965710868 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:17 - global_seqlen/min:158486.0 - global_seqlen/max:158486.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158486.0 - global_seqlen/balanced_max:158486.0 - global_seqlen/mean:158486.0 - actor/entropy:0.16649845242500305 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8045270442962646 - training/rollout_probs_diff_mean:0.005141432862728834 - training/rollout_probs_diff_std:0.01583266817033291 - training/rollout_actor_probs_pearson_corr:0.9968195557594299 - rollout_corr/training_ppl:1.189035415649414 - rollout_corr/training_log_ppl:0.16992178559303284 - rollout_corr/kl:0.0010686807800084352 - rollout_corr/k3_kl:0.0010587231954559684 - rollout_corr/rollout_ppl:1.1877080202102661 - rollout_corr/rollout_log_ppl:0.16886617243289948 - rollout_corr/log_ppl_diff:0.0010556061752140522 - rollout_corr/log_ppl_abs_diff:0.0033748243004083633 - rollout_corr/log_ppl_diff_max:0.032344549894332886 - rollout_corr/log_ppl_diff_min:-0.014729447662830353 - rollout_corr/ppl_ratio:1.0010660886764526 - rollout_corr/chi2_token:0.002133488655090332 - rollout_corr/chi2_seq:0.28585827350616455 - actor/pg_clipfrac:0.003445930201754284 - actor/ppo_kl:3.215687418863429e-05 - actor/pg_clipfrac_lower:1.5338670588486515e-05 - actor/pg_loss:0.0018913421081379056 - actor/loss:0.0018913429230451584 - actor/grad_norm:1.5869140625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.00488615036011 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:17 - training/epoch:0 - critic/score/mean:0.31640625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.31640625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.009219389408826828 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.009219389408826828 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:122.615234375 - response_length/max:128.0 - response_length/min:53.0 - response_length/clip_ratio:0.7822265625 - response_length_non_aborted/mean:122.615234375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:53.0 - response_length_non_aborted/clip_ratio:0.7822265625 - response/aborted_ratio:0.0 - prompt_length/mean:32.15625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0234375 - timing_s/gen:8.19737255400105 - timing_s/old_log_prob:8.825168148003286 - timing_s/adv:0.18529251599829877 - timing_s/update_actor:27.69223388699902 - timing_s/update_weights:1.88338686300267 - timing_s/step:46.959053526006755 - timing_per_token_ms/adv:0.0011691412238197618 - timing_per_token_ms/update_actor:0.17472984293249258 - timing_per_token_ms/gen:0.06528753686743217 - perf/total_num_tokens:158486 - perf/time_per_step:46.959053526006755 - perf/throughput:3374.982843558115 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TransferQueueController pid=2438) 2026-11-01 17:29:38,381 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 29465, Total req/min: 5892.98, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.00 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=11060, req/min=2211.99, avg_time=0.000235s, max_time=0.023539s, min_time=0.000063s; CLEAR_META: req_count=15, req/min=3.00, avg_time=0.001719s, max_time=0.003607s, min_time=0.000830s; SET_CUSTOM_META: req_count=11009, req/min=2201.79, avg_time=0.000009s, max_time=0.000629s, min_time=0.000003s; KV_LIST: req_count=39, req/min=7.80, avg_time=0.000208s, max_time=0.000715s, min_time=0.000048s; NOTIFY_DATA_UPDATE: req_count=7336, req/min=1467.19, avg_time=0.000075s, max_time=0.000822s, min_time=0.000030s; KV_RETRIEVE_KEYS: req_count=6, req/min=1.20, avg_time=0.000106s, max_time=0.000115s, min_time=0.000098s (SimpleStorageUnit pid=2599) Per-operation statistics: CLEAR_DATA: req_count=15, req/min=3.00, avg_time=0.001653s, max_time=0.003576s, min_time=0.000387s; PUT_DATA: req_count=958, req/min=191.60, avg_time=0.000608s, max_time=0.001540s, min_time=0.000319s; GET_DATA: req_count=32, req/min=6.40, avg_time=0.000610s, max_time=0.001191s, min_time=0.000404s (TaskRunnerV1 pid=2351) Training Progress: 13%|█▎ | 18/140 [14:47<1:38:11, 48.29s/it] (SimpleStorageUnit pid=2583) 2026-11-01 17:29:38,574 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1001, Total req/min: 200.20, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.00 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=15, req/min=3.00, avg_time=0.001206s, max_time=0.002120s, min_time=0.000389s; PUT_DATA: req_count=954, req/min=190.80, avg_time=0.000582s, max_time=0.002234s, min_time=0.000320s; GET_DATA: req_count=32, req/min=6.40, avg_time=0.000576s, max_time=0.001213s, min_time=0.000402s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 14%|█▎ | 19/140 [15:34<1:36:42, 47.96s/it] (TaskRunnerV1 pid=2351) Training Progress: 14%|█▍ | 20/140 [16:30<1:40:23, 50.20s/it] (TaskRunnerV1 pid=2351) step:18 - global_seqlen/min:158619.0 - global_seqlen/max:158619.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158619.0 - global_seqlen/balanced_max:158619.0 - global_seqlen/mean:158619.0 - actor/entropy:0.17113721370697021 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.99882572889328 - training/rollout_probs_diff_mean:0.005007949657738209 - training/rollout_probs_diff_std:0.01544660609215498 - training/rollout_actor_probs_pearson_corr:0.9971190094947815 - rollout_corr/training_ppl:1.1952250003814697 - rollout_corr/training_log_ppl:0.17472197115421295 - rollout_corr/kl:0.0011726109078153968 - rollout_corr/k3_kl:0.0010146601125597954 - rollout_corr/rollout_ppl:1.193722128868103 - rollout_corr/rollout_log_ppl:0.1735108345746994 - rollout_corr/log_ppl_diff:0.0012111554387956858 - rollout_corr/log_ppl_abs_diff:0.0033360968809574842 - rollout_corr/log_ppl_diff_max:0.05524203181266785 - rollout_corr/log_ppl_diff_min:-0.0152721107006073 - rollout_corr/ppl_ratio:1.0012235641479492 - rollout_corr/chi2_token:0.0015206336975097656 - rollout_corr/chi2_seq:0.17587792873382568 - actor/pg_clipfrac:0.003133491918561049 - actor/ppo_kl:-0.00015126249005940434 - actor/pg_clipfrac_lower:1.1194698345207144e-05 - actor/pg_loss:0.0016618701629340649 - actor/loss:0.0016618685331195593 - actor/grad_norm:1.564453125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.90027713775635 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:18 - training/epoch:0 - critic/score/mean:0.2666015625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2666015625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.009068563580513 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.009068563580513 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:122.1513671875 - response_length/max:128.0 - response_length/min:32.0 - response_length/clip_ratio:0.791015625 - response_length_non_aborted/mean:122.1513671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:32.0 - response_length_non_aborted/clip_ratio:0.791015625 - response/aborted_ratio:0.0 - prompt_length/mean:32.75 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.187612895999337 - timing_s/old_log_prob:8.809406833002868 - timing_s/adv:0.1878809990012087 - timing_s/update_actor:27.573088265999104 - timing_s/update_weights:1.8557116899974062 - timing_s/step:46.79919583599985 - timing_per_token_ms/adv:0.001184479784900981 - timing_per_token_ms/update_actor:0.17383219075898287 - timing_per_token_ms/gen:0.06545743942821436 - perf/total_num_tokens:158619 - perf/time_per_step:46.79919583599985 - perf/throughput:3389.3531110204203 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:19 - global_seqlen/min:159647.0 - global_seqlen/max:159647.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159647.0 - global_seqlen/balanced_max:159647.0 - global_seqlen/mean:159647.0 - actor/entropy:0.16078148782253265 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6551046967506409 - training/rollout_probs_diff_mean:0.004890823271125555 - training/rollout_probs_diff_std:0.015046036802232265 - training/rollout_actor_probs_pearson_corr:0.9970410466194153 - rollout_corr/training_ppl:1.1823631525039673 - rollout_corr/training_log_ppl:0.16468220949172974 - rollout_corr/kl:0.0008574216044507921 - rollout_corr/k3_kl:0.0009737764485180378 - rollout_corr/rollout_ppl:1.1813526153564453 - rollout_corr/rollout_log_ppl:0.16385437548160553 - rollout_corr/log_ppl_diff:0.0008278356399387121 - rollout_corr/log_ppl_abs_diff:0.003281565383076668 - rollout_corr/log_ppl_diff_max:0.041472479701042175 - rollout_corr/log_ppl_diff_min:-0.020124629139900208 - rollout_corr/ppl_ratio:1.0008375644683838 - rollout_corr/chi2_token:0.002079486846923828 - rollout_corr/chi2_seq:0.3488885164260864 - actor/pg_clipfrac:0.0031762407040029457 - actor/ppo_kl:0.0002532675240824271 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0020338176982477307 - actor/loss:0.0020338173490017653 - actor/grad_norm:1.5458984375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.97160577774048 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:19 - training/epoch:0 - critic/score/mean:0.2939453125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2939453125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006425490137189627 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006425490137189627 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:123.2880859375 - response_length/max:128.0 - response_length/min:49.0 - response_length/clip_ratio:0.7998046875 - response_length_non_aborted/mean:123.2880859375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:49.0 - response_length_non_aborted/clip_ratio:0.7998046875 - response/aborted_ratio:0.0 - prompt_length/mean:32.6171875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.208232698001666 - timing_s/old_log_prob:8.66348562300118 - timing_s/adv:0.3953885610026191 - timing_s/update_actor:27.77695016800135 - timing_s/update_weights:1.842351002000214 - timing_s/step:47.05429687200376 - timing_per_token_ms/adv:0.002476642599000414 - timing_per_token_ms/update_actor:0.17398980355409965 - timing_per_token_ms/gen:0.06501724950297168 - perf/total_num_tokens:159647 - perf/time_per_step:47.05429687200376 - perf/throughput:3392.825110834593 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:20 - global_seqlen/min:157719.0 - global_seqlen/max:157719.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157719.0 - global_seqlen/balanced_max:157719.0 - global_seqlen/mean:157719.0 - actor/entropy:0.16101518273353577 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8017211556434631 - training/rollout_probs_diff_mean:0.004941312130540609 - training/rollout_probs_diff_std:0.015313408337533474 - training/rollout_actor_probs_pearson_corr:0.9969057440757751 - rollout_corr/training_ppl:1.1792157888412476 - rollout_corr/training_log_ppl:0.16205748915672302 - rollout_corr/kl:0.000928406894672662 - rollout_corr/k3_kl:0.0010011536069214344 - rollout_corr/rollout_ppl:1.1781260967254639 - rollout_corr/rollout_log_ppl:0.16117912530899048 - rollout_corr/log_ppl_diff:0.0008783634984865785 - rollout_corr/log_ppl_abs_diff:0.0032157383393496275 - rollout_corr/log_ppl_diff_max:0.06201288849115372 - rollout_corr/log_ppl_diff_min:-0.015351176261901855 - rollout_corr/ppl_ratio:1.0008894205093384 - rollout_corr/chi2_token:0.0020945072174072266 - rollout_corr/chi2_seq:0.2821507453918457 - actor/pg_clipfrac:0.0028075923861858123 - actor/ppo_kl:0.00019967607765162634 - actor/pg_clipfrac_lower:7.573004040750675e-06 - actor/pg_loss:0.0003281041426816955 - actor/loss:0.00032810308039188385 - actor/grad_norm:1.57421875 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.91743469238281 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.331) - val-core/char_count/acc/mean@1:np.float64(0.331) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:20 - training/epoch:0 - critic/score/mean:0.3125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007867367006838322 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007867367006838322 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:121.9208984375 - response_length/max:128.0 - response_length/min:38.0 - response_length/clip_ratio:0.78515625 - response_length_non_aborted/mean:121.9208984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:38.0 - response_length_non_aborted/clip_ratio:0.78515625 - response/aborted_ratio:0.0 - prompt_length/mean:32.1015625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.178864298999542 - timing_s/old_log_prob:8.63669108600152 - timing_s/adv:0.38785438599734334 - timing_s/update_actor:27.3765880150022 - timing_s/update_weights:1.8535944010000094 - timing_s/step:46.60197536100168 - timing_s/testing:8.692768576001981 - timing_per_token_ms/adv:0.0024591481431998894 - timing_per_token_ms/update_actor:0.17357825002062022 - timing_per_token_ms/gen:0.06551109997836986 - perf/total_num_tokens:157719 - perf/time_per_step:46.60197536100168 - perf/throughput:3384.3844338835756 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 15%|█▌ | 21/140 [17:17<1:37:32, 49.18s/it] (TaskRunnerV1 pid=2351) Training Progress: 16%|█▌ | 22/140 [18:03<1:35:21, 48.49s/it] (TaskRunnerV1 pid=2351) Training Progress: 16%|█▋ | 23/140 [18:51<1:33:49, 48.11s/it] (TaskRunnerV1 pid=2351) step:21 - global_seqlen/min:157688.0 - global_seqlen/max:157688.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157688.0 - global_seqlen/balanced_max:157688.0 - global_seqlen/mean:157688.0 - actor/entropy:0.16391873359680176 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.985840380191803 - training/rollout_probs_diff_mean:0.004943474195897579 - training/rollout_probs_diff_std:0.016458982601761818 - training/rollout_actor_probs_pearson_corr:0.9965984225273132 - rollout_corr/training_ppl:1.1870478391647339 - rollout_corr/training_log_ppl:0.16833370923995972 - rollout_corr/kl:0.0012787507148459554 - rollout_corr/k3_kl:0.0011261546751484275 - rollout_corr/rollout_ppl:1.1854727268218994 - rollout_corr/rollout_log_ppl:0.16704560816287994 - rollout_corr/log_ppl_diff:0.001288090948946774 - rollout_corr/log_ppl_abs_diff:0.003408274380490184 - rollout_corr/log_ppl_diff_max:0.050126731395721436 - rollout_corr/log_ppl_diff_min:-0.014394253492355347 - rollout_corr/ppl_ratio:1.0013017654418945 - rollout_corr/chi2_token:0.001611948013305664 - rollout_corr/chi2_seq:0.18423950672149658 - actor/pg_clipfrac:0.003239611141907517 - actor/ppo_kl:0.00020035981405423323 - actor/pg_clipfrac_lower:1.532074808589338e-05 - actor/pg_loss:0.0013072311503492529 - actor/loss:0.0013072288129478693 - actor/grad_norm:1.556640625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.980080127716064 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:21 - training/epoch:0 - critic/score/mean:0.294921875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.294921875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.010013089515268803 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.010013089515268803 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:121.578125 - response_length/max:128.0 - response_length/min:32.0 - response_length/clip_ratio:0.73828125 - response_length_non_aborted/mean:121.578125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:32.0 - response_length_non_aborted/clip_ratio:0.73828125 - response/aborted_ratio:0.0 - prompt_length/mean:32.4140625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.202760243002558 - timing_s/old_log_prob:8.84206885399908 - timing_s/adv:0.1869857839992619 - timing_s/update_actor:27.430515986998216 - timing_s/update_weights:1.857195149998006 - timing_s/step:46.68645869700413 - timing_per_token_ms/adv:0.001185795900761389 - timing_per_token_ms/update_actor:0.17395436550021698 - timing_per_token_ms/gen:0.06588774131701065 - perf/total_num_tokens:157688 - perf/time_per_step:46.68645869700413 - perf/throughput:3377.5960824828817 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:22 - global_seqlen/min:157837.0 - global_seqlen/max:157837.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157837.0 - global_seqlen/balanced_max:157837.0 - global_seqlen/mean:157837.0 - actor/entropy:0.16056719422340393 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5837955474853516 - training/rollout_probs_diff_mean:0.004937994293868542 - training/rollout_probs_diff_std:0.015467443503439426 - training/rollout_actor_probs_pearson_corr:0.9969344735145569 - rollout_corr/training_ppl:1.1827340126037598 - rollout_corr/training_log_ppl:0.16490961611270905 - rollout_corr/kl:0.0009003891609609127 - rollout_corr/k3_kl:0.0010261160787194967 - rollout_corr/rollout_ppl:1.1816238164901733 - rollout_corr/rollout_log_ppl:0.16401836276054382 - rollout_corr/log_ppl_diff:0.0008912591729313135 - rollout_corr/log_ppl_abs_diff:0.003163395682349801 - rollout_corr/log_ppl_diff_max:0.021600395441055298 - rollout_corr/log_ppl_diff_min:-0.02169506996870041 - rollout_corr/ppl_ratio:1.000900149345398 - rollout_corr/chi2_token:0.002777576446533203 - rollout_corr/chi2_seq:0.5448888540267944 - actor/pg_clipfrac:0.0034113152360077947 - actor/ppo_kl:-2.2168342184156263e-07 - actor/pg_clipfrac_lower:7.650874977116473e-06 - actor/pg_loss:-0.0019103352642559912 - actor/loss:-0.0019103363156318665 - actor/grad_norm:1.5556640625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.01127099990845 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:22 - training/epoch:0 - critic/score/mean:0.349609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.349609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005855774041265249 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005855774041265249 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:121.8330078125 - response_length/max:128.0 - response_length/min:48.0 - response_length/clip_ratio:0.7529296875 - response_length_non_aborted/mean:121.8330078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:48.0 - response_length_non_aborted/clip_ratio:0.7529296875 - response/aborted_ratio:0.0 - prompt_length/mean:32.3046875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.189962455995556 - timing_s/old_log_prob:8.604116788999818 - timing_s/adv:0.3832421440019971 - timing_s/update_actor:27.557314596997458 - timing_s/update_weights:1.8408602549970965 - timing_s/step:46.74473836099787 - timing_per_token_ms/adv:0.002428088116233818 - timing_per_token_ms/update_actor:0.1745935021382658 - timing_per_token_ms/gen:0.06564731803422297 - perf/total_num_tokens:157837 - perf/time_per_step:46.74473836099787 - perf/throughput:3376.572541300039 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:23 - global_seqlen/min:159109.0 - global_seqlen/max:159109.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:159109.0 - global_seqlen/balanced_max:159109.0 - global_seqlen/mean:159109.0 - actor/entropy:0.16439352929592133 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.727885901927948 - training/rollout_probs_diff_mean:0.00501282699406147 - training/rollout_probs_diff_std:0.01619558036327362 - training/rollout_actor_probs_pearson_corr:0.9966962933540344 - rollout_corr/training_ppl:1.1857831478118896 - rollout_corr/training_log_ppl:0.16712860763072968 - rollout_corr/kl:0.0010848201345652342 - rollout_corr/k3_kl:0.0010207307059317827 - rollout_corr/rollout_ppl:1.184432029724121 - rollout_corr/rollout_log_ppl:0.16603311896324158 - rollout_corr/log_ppl_diff:0.0010954904137179255 - rollout_corr/log_ppl_abs_diff:0.0033114177640527487 - rollout_corr/log_ppl_diff_max:0.04143562912940979 - rollout_corr/log_ppl_diff_min:-0.020034804940223694 - rollout_corr/ppl_ratio:1.0011063814163208 - rollout_corr/chi2_token:0.0018475055694580078 - rollout_corr/chi2_seq:0.2173004150390625 - actor/pg_clipfrac:0.0036990212320233695 - actor/ppo_kl:-0.00013633592512481604 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.003001778502948582 - actor/loss:-0.0030017774552106857 - actor/grad_norm:1.7080078125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.960317611694336 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:23 - training/epoch:0 - critic/score/mean:0.2568359375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2568359375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005959256552159786 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005959256552159786 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:122.7314453125 - response_length/max:128.0 - response_length/min:46.0 - response_length/clip_ratio:0.7978515625 - response_length_non_aborted/mean:122.7314453125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:46.0 - response_length_non_aborted/clip_ratio:0.7978515625 - response/aborted_ratio:0.0 - prompt_length/mean:32.6484375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.181320039002458 - timing_s/old_log_prob:8.703306473995326 - timing_s/adv:0.38446822500554845 - timing_s/update_actor:27.829384735996427 - timing_s/update_weights:1.8357369209988974 - timing_s/step:47.100336406998395 - timing_per_token_ms/adv:0.002416382637095001 - timing_per_token_ms/update_actor:0.17490767169673888 - timing_per_token_ms/gen:0.0650979896003442 - perf/total_num_tokens:159109 - perf/time_per_step:47.100336406998395 - perf/throughput:3378.0862757565956 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TransferQueueController pid=2438) 2026-11-01 17:34:39,267 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 31466, Total req/min: 6274.65, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.01 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=11714, req/min=2335.89, avg_time=0.000245s, max_time=0.065833s, min_time=0.000064s; NOTIFY_DATA_UPDATE: req_count=8013, req/min=1597.88, avg_time=0.000076s, max_time=0.000867s, min_time=0.000030s; SET_CUSTOM_META: req_count=11682, req/min=2329.51, avg_time=0.000009s, max_time=0.001465s, min_time=0.000003s; KV_LIST: req_count=36, req/min=7.18, avg_time=0.000227s, max_time=0.000818s, min_time=0.000050s; CLEAR_META: req_count=15, req/min=2.99, avg_time=0.001713s, max_time=0.003834s, min_time=0.000751s; KV_RETRIEVE_KEYS: req_count=6, req/min=1.20, avg_time=0.000122s, max_time=0.000196s, min_time=0.000099s (SimpleStorageUnit pid=2599) Per-operation statistics: PUT_DATA: req_count=984, req/min=196.35, avg_time=0.000576s, max_time=0.004442s, min_time=0.000308s; CLEAR_DATA: req_count=15, req/min=2.99, avg_time=0.001263s, max_time=0.002951s, min_time=0.000431s; GET_DATA: req_count=33, req/min=6.58, avg_time=0.000584s, max_time=0.000928s, min_time=0.000393s (TaskRunnerV1 pid=2351) Training Progress: 17%|█▋ | 24/140 [19:38<1:32:18, 47.75s/it] (SimpleStorageUnit pid=2583) 2026-11-01 17:34:39,256 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1050, Total req/min: 209.52, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.01 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: PUT_DATA: req_count=1002, req/min=199.95, avg_time=0.000596s, max_time=0.003007s, min_time=0.000323s; CLEAR_DATA: req_count=15, req/min=2.99, avg_time=0.001286s, max_time=0.004305s, min_time=0.000407s; GET_DATA: req_count=33, req/min=6.59, avg_time=0.000598s, max_time=0.001240s, min_time=0.000401s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 18%|█▊ | 25/140 [20:35<1:37:16, 50.75s/it] (TaskRunnerV1 pid=2351) Training Progress: 19%|█▊ | 26/140 [21:22<1:33:57, 49.45s/it] (TaskRunnerV1 pid=2351) step:24 - global_seqlen/min:157972.0 - global_seqlen/max:157972.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157972.0 - global_seqlen/balanced_max:157972.0 - global_seqlen/mean:157972.0 - actor/entropy:0.1528375893831253 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9005656838417053 - training/rollout_probs_diff_mean:0.004696883261203766 - training/rollout_probs_diff_std:0.015141437761485577 - training/rollout_actor_probs_pearson_corr:0.996859610080719 - rollout_corr/training_ppl:1.173482060432434 - rollout_corr/training_log_ppl:0.1572272777557373 - rollout_corr/kl:0.0010357302380725741 - rollout_corr/k3_kl:0.0009713491308502853 - rollout_corr/rollout_ppl:1.1722038984298706 - rollout_corr/rollout_log_ppl:0.15617458522319794 - rollout_corr/log_ppl_diff:0.0010526780970394611 - rollout_corr/log_ppl_abs_diff:0.003120194422081113 - rollout_corr/log_ppl_diff_max:0.06435468792915344 - rollout_corr/log_ppl_diff_min:-0.013248130679130554 - rollout_corr/ppl_ratio:1.0010638236999512 - rollout_corr/chi2_token:0.0016756057739257812 - rollout_corr/chi2_seq:0.17025375366210938 - actor/pg_clipfrac:0.003009256705505929 - actor/ppo_kl:4.37832983758805e-05 - actor/pg_clipfrac_lower:2.295194341665289e-05 - actor/pg_loss:-0.0002701208577491343 - actor/loss:-0.0002701221383176744 - actor/grad_norm:1.439453125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.95460653305054 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:24 - training/epoch:0 - critic/score/mean:0.2939453125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2939453125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007606783881783485 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007606783881783485 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:121.82421875 - response_length/max:128.0 - response_length/min:36.0 - response_length/clip_ratio:0.767578125 - response_length_non_aborted/mean:121.82421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:36.0 - response_length_non_aborted/clip_ratio:0.767578125 - response/aborted_ratio:0.0 - prompt_length/mean:32.4453125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.194227255000442 - timing_s/old_log_prob:8.640957853996952 - timing_s/adv:0.1836577239955659 - timing_s/update_actor:27.7301139409974 - timing_s/update_weights:1.8565434910051408 - timing_s/step:46.77237000199966 - timing_per_token_ms/adv:0.001162596687992593 - timing_per_token_ms/update_actor:0.17553815828752817 - timing_per_token_ms/gen:0.06568624150287333 - perf/total_num_tokens:157972 - perf/time_per_step:46.77237000199966 - perf/throughput:3377.464088162439 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:25 - global_seqlen/min:157922.0 - global_seqlen/max:157922.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157922.0 - global_seqlen/balanced_max:157922.0 - global_seqlen/mean:157922.0 - actor/entropy:0.15913891792297363 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8514849543571472 - training/rollout_probs_diff_mean:0.00481199286878109 - training/rollout_probs_diff_std:0.015728527680039406 - training/rollout_actor_probs_pearson_corr:0.9967331290245056 - rollout_corr/training_ppl:1.178812026977539 - rollout_corr/training_log_ppl:0.1615656316280365 - rollout_corr/kl:0.0013396341819316149 - rollout_corr/k3_kl:0.0010522266384214163 - rollout_corr/rollout_ppl:1.1772080659866333 - rollout_corr/rollout_log_ppl:0.16025319695472717 - rollout_corr/log_ppl_diff:0.0013124353718012571 - rollout_corr/log_ppl_abs_diff:0.0032997638918459415 - rollout_corr/log_ppl_diff_max:0.09559985995292664 - rollout_corr/log_ppl_diff_min:-0.015088841319084167 - rollout_corr/ppl_ratio:1.0013264417648315 - rollout_corr/chi2_token:0.0014700889587402344 - rollout_corr/chi2_seq:0.20283782482147217 - actor/pg_clipfrac:0.003225722185258443 - actor/ppo_kl:-0.00025180328126831836 - actor/pg_clipfrac_lower:7.75193802837748e-06 - actor/pg_loss:0.002463698503561318 - actor/loss:0.0024636967573314905 - actor/grad_norm:1.6796875 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.07124996185303 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.331) - val-core/char_count/acc/mean@1:np.float64(0.331) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:25 - training/epoch:0 - critic/score/mean:0.3388671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3388671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.012730874121189117 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.012730874121189117 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:121.876953125 - response_length/max:128.0 - response_length/min:51.0 - response_length/clip_ratio:0.7548828125 - response_length_non_aborted/mean:121.876953125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:51.0 - response_length_non_aborted/clip_ratio:0.7548828125 - response/aborted_ratio:0.0 - prompt_length/mean:32.34375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:8.185218585000257 - timing_s/old_log_prob:8.592574406000494 - timing_s/adv:0.18687346799561055 - timing_s/update_actor:27.582982483996602 - timing_s/update_weights:1.8574717929950566 - timing_s/step:46.58173338600318 - timing_s/testing:11.054724904002796 - timing_per_token_ms/adv:0.001183327642732555 - timing_per_token_ms/update_actor:0.17466206408224694 - timing_per_token_ms/gen:0.06558563632794552 - perf/total_num_tokens:157922 - perf/time_per_step:46.58173338600318 - perf/throughput:3390.213041051242 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:26 - global_seqlen/min:155815.0 - global_seqlen/max:155815.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155815.0 - global_seqlen/balanced_max:155815.0 - global_seqlen/mean:155815.0 - actor/entropy:0.1656489074230194 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9131549000740051 - training/rollout_probs_diff_mean:0.00508819380775094 - training/rollout_probs_diff_std:0.015931181609630585 - training/rollout_actor_probs_pearson_corr:0.9967937469482422 - rollout_corr/training_ppl:1.1897917985916138 - rollout_corr/training_log_ppl:0.17026367783546448 - rollout_corr/kl:0.0013581870589405298 - rollout_corr/k3_kl:0.0010131935123354197 - rollout_corr/rollout_ppl:1.1880649328231812 - rollout_corr/rollout_log_ppl:0.16885748505592346 - rollout_corr/log_ppl_diff:0.0014061828842386603 - rollout_corr/log_ppl_abs_diff:0.003433038480579853 - rollout_corr/log_ppl_diff_max:0.0340987890958786 - rollout_corr/log_ppl_diff_min:-0.01700606942176819 - rollout_corr/ppl_ratio:1.001417875289917 - rollout_corr/chi2_token:0.0013076066970825195 - rollout_corr/chi2_seq:0.13329291343688965 - actor/pg_clipfrac:0.0025891966264074045 - actor/ppo_kl:-0.0001769624023305975 - actor/pg_clipfrac_lower:1.5151514768755684e-05 - actor/pg_loss:-0.0009709511941764504 - actor/loss:-0.0009709516307339072 - actor/grad_norm:2.140625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.045517921447754 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:26 - training/epoch:0 - critic/score/mean:0.3291015625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3291015625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.011652975343167782 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.011652975343167782 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:119.7333984375 - response_length/max:128.0 - response_length/min:32.0 - response_length/clip_ratio:0.72265625 - response_length_non_aborted/mean:119.7333984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:32.0 - response_length_non_aborted/clip_ratio:0.72265625 - response/aborted_ratio:0.0 - prompt_length/mean:32.4296875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.189229241004796 - timing_s/old_log_prob:8.577310191001743 - timing_s/adv:0.38668197199876886 - timing_s/update_actor:27.10056087299745 - timing_s/update_weights:1.8529280989969266 - timing_s/step:46.28034159500385 - timing_per_token_ms/adv:0.0024816736000947844 - timing_per_token_ms/update_actor:0.17392780459517665 - timing_per_token_ms/gen:0.0667925097343936 - perf/total_num_tokens:155815 - perf/time_per_step:46.28034159500385 - perf/throughput:3366.764259510584 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 19%|█▉ | 27/140 [22:09<1:31:41, 48.69s/it] (TaskRunnerV1 pid=2351) Training Progress: 20%|██ | 28/140 [22:56<1:29:58, 48.20s/it] (TaskRunnerV1 pid=2351) Training Progress: 21%|██ | 29/140 [23:42<1:27:54, 47.52s/it] (TaskRunnerV1 pid=2351) step:27 - global_seqlen/min:157749.0 - global_seqlen/max:157749.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157749.0 - global_seqlen/balanced_max:157749.0 - global_seqlen/mean:157749.0 - actor/entropy:0.15282654762268066 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8445637226104736 - training/rollout_probs_diff_mean:0.004761972930282354 - training/rollout_probs_diff_std:0.01558681484311819 - training/rollout_actor_probs_pearson_corr:0.996752142906189 - rollout_corr/training_ppl:1.170972228050232 - rollout_corr/training_log_ppl:0.15521420538425446 - rollout_corr/kl:0.000756822875700891 - rollout_corr/k3_kl:0.0009301853715442121 - rollout_corr/rollout_ppl:1.170059084892273 - rollout_corr/rollout_log_ppl:0.15446963906288147 - rollout_corr/log_ppl_diff:0.0007445861701853573 - rollout_corr/log_ppl_abs_diff:0.003138683270663023 - rollout_corr/log_ppl_diff_max:0.01854729652404785 - rollout_corr/log_ppl_diff_min:-0.01574847102165222 - rollout_corr/ppl_ratio:1.0007532835006714 - rollout_corr/chi2_token:0.0023300647735595703 - rollout_corr/chi2_seq:0.31278061866760254 - actor/pg_clipfrac:0.0032789650479874886 - actor/ppo_kl:6.843134982167007e-05 - actor/pg_clipfrac_lower:2.3174470091665476e-05 - actor/pg_loss:-0.00047504536632914096 - actor/loss:-0.0004750448279082775 - actor/grad_norm:1.56640625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.06173801422119 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:27 - training/epoch:0 - critic/score/mean:0.3681640625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3681640625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00990599300712347 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.00990599300712347 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:121.8330078125 - response_length/max:128.0 - response_length/min:53.0 - response_length/clip_ratio:0.728515625 - response_length_non_aborted/mean:121.8330078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:53.0 - response_length_non_aborted/clip_ratio:0.728515625 - response/aborted_ratio:0.0 - prompt_length/mean:32.21875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.20228885099641 - timing_s/old_log_prob:8.572461954994651 - timing_s/adv:0.1826214210013859 - timing_s/update_actor:27.797957412993128 - timing_s/update_weights:1.8545945869991556 - timing_s/step:46.780383732999326 - timing_per_token_ms/adv:0.0011576708632155253 - timing_per_token_ms/update_actor:0.17621637799918305 - timing_per_token_ms/gen:0.06574612126771573 - perf/total_num_tokens:157749 - perf/time_per_step:46.780383732999326 - perf/throughput:3372.1185550840696 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:28 - global_seqlen/min:158542.0 - global_seqlen/max:158542.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158542.0 - global_seqlen/balanced_max:158542.0 - global_seqlen/mean:158542.0 - actor/entropy:0.16063809394836426 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.776964545249939 - training/rollout_probs_diff_mean:0.004955651704221964 - training/rollout_probs_diff_std:0.015878506004810333 - training/rollout_actor_probs_pearson_corr:0.9967760443687439 - rollout_corr/training_ppl:1.1822055578231812 - rollout_corr/training_log_ppl:0.16401807963848114 - rollout_corr/kl:0.0008943339926190674 - rollout_corr/k3_kl:0.000991438515484333 - rollout_corr/rollout_ppl:1.1810935735702515 - rollout_corr/rollout_log_ppl:0.16310793161392212 - rollout_corr/log_ppl_diff:0.0009101637988351285 - rollout_corr/log_ppl_abs_diff:0.0032567325979471207 - rollout_corr/log_ppl_diff_max:0.06227179616689682 - rollout_corr/log_ppl_diff_min:-0.015792101621627808 - rollout_corr/ppl_ratio:1.0009210109710693 - rollout_corr/chi2_token:0.0020769834518432617 - rollout_corr/chi2_seq:0.31396496295928955 - actor/pg_clipfrac:0.0037836512443997585 - actor/ppo_kl:0.0002308006769453641 - actor/pg_clipfrac_lower:7.597860530950129e-06 - actor/pg_loss:0.0014092376513872296 - actor/loss:0.0014092379715293646 - actor/grad_norm:1.6162109375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.99953317642212 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:28 - training/epoch:0 - critic/score/mean:0.3310546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3310546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.010593459010124207 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.010593459010124207 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:122.412109375 - response_length/max:128.0 - response_length/min:49.0 - response_length/clip_ratio:0.76171875 - response_length_non_aborted/mean:122.412109375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:49.0 - response_length_non_aborted/clip_ratio:0.76171875 - response/aborted_ratio:0.0 - prompt_length/mean:32.4140625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.181994523998583 - timing_s/old_log_prob:8.619398031994933 - timing_s/adv:0.1832654189929599 - timing_s/update_actor:27.75897522699961 - timing_s/update_weights:1.8540039649960818 - timing_s/step:46.76586335599859 - timing_per_token_ms/adv:0.0011559423937692213 - timing_per_token_ms/update_actor:0.17508909454276855 - timing_per_token_ms/gen:0.06527319125647055 - perf/total_num_tokens:158542 - perf/time_per_step:46.76586335599859 - perf/throughput:3390.122380359392 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:29 - global_seqlen/min:153304.0 - global_seqlen/max:153304.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:153304.0 - global_seqlen/balanced_max:153304.0 - global_seqlen/mean:153304.0 - actor/entropy:0.1605193316936493 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8626787066459656 - training/rollout_probs_diff_mean:0.00504913367331028 - training/rollout_probs_diff_std:0.01580905355513096 - training/rollout_actor_probs_pearson_corr:0.9967293739318848 - rollout_corr/training_ppl:1.1830554008483887 - rollout_corr/training_log_ppl:0.1652597337961197 - rollout_corr/kl:0.0008604638278484344 - rollout_corr/k3_kl:0.0010110917501151562 - rollout_corr/rollout_ppl:1.1819452047348022 - rollout_corr/rollout_log_ppl:0.16437473893165588 - rollout_corr/log_ppl_diff:0.0008849860751070082 - rollout_corr/log_ppl_abs_diff:0.0033764005638659 - rollout_corr/log_ppl_diff_max:0.02641695737838745 - rollout_corr/log_ppl_diff_min:-0.016600623726844788 - rollout_corr/ppl_ratio:1.0008951425552368 - rollout_corr/chi2_token:0.0026090145111083984 - rollout_corr/chi2_seq:0.30902111530303955 - actor/pg_clipfrac:0.0033027148213780797 - actor/ppo_kl:0.00010120026248235565 - actor/pg_clipfrac_lower:1.5161178453126922e-05 - actor/pg_loss:-0.0004895113816019148 - actor/loss:-0.0004895105957984924 - actor/grad_norm:1.736328125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.05476903915405 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:29 - training/epoch:0 - critic/score/mean:0.3779296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3779296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.011087659746408463 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.011087659746408463 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:117.890625 - response_length/max:128.0 - response_length/min:26.0 - response_length/clip_ratio:0.66015625 - response_length_non_aborted/mean:117.890625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:26.0 - response_length_non_aborted/clip_ratio:0.66015625 - response/aborted_ratio:0.0 - prompt_length/mean:31.8203125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.201419004995842 - timing_s/old_log_prob:8.371605824999278 - timing_s/adv:0.1834364349997486 - timing_s/update_actor:27.006395169002644 - timing_s/update_weights:1.8671339189968421 - timing_s/step:45.7985894950034 - timing_per_token_ms/adv:0.001196553481968824 - timing_per_token_ms/update_actor:0.1761623647719736 - timing_per_token_ms/gen:0.06793753317590989 - perf/total_num_tokens:153304 - perf/time_per_step:45.7985894950034 - perf/throughput:3347.3519968715495 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TransferQueueController pid=2438) 2026-11-01 17:40:00,078 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 28680, Total req/min: 5363.91, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.35 minutes; (TransferQueueController pid=2438) Per-operation statistics: SET_CUSTOM_META: req_count=10708, req/min=2002.67, avg_time=0.000009s, max_time=0.000439s, min_time=0.000003s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.31, avg_time=0.000110s, max_time=0.000124s, min_time=0.000099s; KV_RETRIEVE_META: req_count=10751, req/min=2010.72, avg_time=0.000248s, max_time=0.017089s, min_time=0.000063s; NOTIFY_DATA_UPDATE: req_count=7164, req/min=1339.85, avg_time=0.000075s, max_time=0.000727s, min_time=0.000030s; CLEAR_META: req_count=14, req/min=2.62, avg_time=0.001674s, max_time=0.003299s, min_time=0.000878s; KV_LIST: req_count=36, req/min=6.73, avg_time=0.000237s, max_time=0.000995s, min_time=0.000036s (SimpleStorageUnit pid=2599) Per-operation statistics: GET_DATA: req_count=35, req/min=6.49, avg_time=0.000583s, max_time=0.000933s, min_time=0.000415s; PUT_DATA: req_count=891, req/min=165.11, avg_time=0.000591s, max_time=0.001740s, min_time=0.000322s; CLEAR_DATA: req_count=14, req/min=2.59, avg_time=0.001203s, max_time=0.002087s, min_time=0.000436s (TaskRunnerV1 pid=2351) Training Progress: 21%|██▏ | 30/140 [24:37<1:31:25, 49.87s/it] (SimpleStorageUnit pid=2582) 2026-11-01 17:40:04,046 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_e1220d54: [Performance] Total success requests: 971, Total req/min: 179.38, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2582) Time range: last 5.41 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2582) Per-operation statistics: GET_DATA: req_count=35, req/min=6.47, avg_time=0.000615s, max_time=0.001229s, min_time=0.000413s; PUT_DATA: req_count=922, req/min=170.33, avg_time=0.000592s, max_time=0.001392s, min_time=0.000314s; CLEAR_DATA: req_count=14, req/min=2.59, avg_time=0.001413s, max_time=0.003935s, min_time=0.000391s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 22%|██▏ | 31/140 [25:24<1:28:52, 48.92s/it] (TaskRunnerV1 pid=2351) Training Progress: 23%|██▎ | 32/140 [26:10<1:26:46, 48.21s/it] (TaskRunnerV1 pid=2351) step:30 - global_seqlen/min:157050.0 - global_seqlen/max:157050.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157050.0 - global_seqlen/balanced_max:157050.0 - global_seqlen/mean:157050.0 - actor/entropy:0.1526092290878296 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9961878657341003 - training/rollout_probs_diff_mean:0.004673470743000507 - training/rollout_probs_diff_std:0.015512768179178238 - training/rollout_actor_probs_pearson_corr:0.996820330619812 - rollout_corr/training_ppl:1.1719869375228882 - rollout_corr/training_log_ppl:0.1561393439769745 - rollout_corr/kl:0.0013051567366346717 - rollout_corr/k3_kl:0.0010498856427147985 - rollout_corr/rollout_ppl:1.1704180240631104 - rollout_corr/rollout_log_ppl:0.15482211112976074 - rollout_corr/log_ppl_diff:0.0013172357575967908 - rollout_corr/log_ppl_abs_diff:0.0032960360404103994 - rollout_corr/log_ppl_diff_max:0.09602940082550049 - rollout_corr/log_ppl_diff_min:-0.012573391199111938 - rollout_corr/ppl_ratio:1.001333236694336 - rollout_corr/chi2_token:0.001212000846862793 - rollout_corr/chi2_seq:0.1787738800048828 - actor/pg_clipfrac:0.0028715157944437424 - actor/ppo_kl:-0.00013558405097076803 - actor/pg_clipfrac_lower:1.6059792566617638e-05 - actor/pg_loss:-0.0005087371537229046 - actor/loss:-0.0005087365861982107 - actor/grad_norm:1.54541015625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.98717927932739 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.365) - val-core/char_count/acc/mean@1:np.float64(0.365) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:30 - training/epoch:0 - critic/score/mean:0.3671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004717184230685234 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004717184230685234 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:120.751953125 - response_length/max:128.0 - response_length/min:40.0 - response_length/clip_ratio:0.69921875 - response_length_non_aborted/mean:120.751953125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:40.0 - response_length_non_aborted/clip_ratio:0.69921875 - response/aborted_ratio:0.0 - prompt_length/mean:32.6171875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.193055098003242 - timing_s/old_log_prob:8.61460180299764 - timing_s/adv:0.22477617299591657 - timing_s/update_actor:27.308863556994766 - timing_s/update_weights:1.8530138470014208 - timing_s/step:46.3622541299992 - timing_s/testing:8.872488531997078 - timing_per_token_ms/adv:0.0014312395606234738 - timing_per_token_ms/update_actor:0.17388642825211567 - timing_per_token_ms/gen:0.06626004931664571 - perf/total_num_tokens:157050 - perf/time_per_step:46.3622541299992 - perf/throughput:3387.4539309420484 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:31 - global_seqlen/min:156396.0 - global_seqlen/max:156396.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156396.0 - global_seqlen/balanced_max:156396.0 - global_seqlen/mean:156396.0 - actor/entropy:0.1481461077928543 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9189216494560242 - training/rollout_probs_diff_mean:0.0046490272507071495 - training/rollout_probs_diff_std:0.01560758613049984 - training/rollout_actor_probs_pearson_corr:0.9966513514518738 - rollout_corr/training_ppl:1.167697787284851 - rollout_corr/training_log_ppl:0.15258948504924774 - rollout_corr/kl:0.0009912977693602443 - rollout_corr/k3_kl:0.0009537928854115307 - rollout_corr/rollout_ppl:1.166541576385498 - rollout_corr/rollout_log_ppl:0.15161411464214325 - rollout_corr/log_ppl_diff:0.0009753532940521836 - rollout_corr/log_ppl_abs_diff:0.0031388485804200172 - rollout_corr/log_ppl_diff_max:0.02853647619485855 - rollout_corr/log_ppl_diff_min:-0.017452508211135864 - rollout_corr/ppl_ratio:1.0009846687316895 - rollout_corr/chi2_token:0.0019180774688720703 - rollout_corr/chi2_seq:0.2236921787261963 - actor/pg_clipfrac:0.002632406108508197 - actor/ppo_kl:-8.305766111031214e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0005315264570526779 - actor/loss:0.0005315253511071205 - actor/grad_norm:1.537109375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.00792455673218 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:31 - training/epoch:0 - critic/score/mean:0.36328125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.36328125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008318721316754818 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008318721316754818 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:120.23046875 - response_length/max:128.0 - response_length/min:51.0 - response_length/clip_ratio:0.720703125 - response_length_non_aborted/mean:120.23046875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:51.0 - response_length_non_aborted/clip_ratio:0.720703125 - response/aborted_ratio:0.0 - prompt_length/mean:32.5 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:8.195529075004742 - timing_s/old_log_prob:8.616512569999031 - timing_s/adv:0.23700619100418407 - timing_s/update_actor:27.50968243299576 - timing_s/update_weights:1.8585133890010184 - timing_s/step:46.584034756997426 - timing_per_token_ms/adv:0.0015154236106050287 - timing_per_token_ms/update_actor:0.17589760884546765 - timing_per_token_ms/gen:0.06656753854092679 - perf/total_num_tokens:156396 - perf/time_per_step:46.584034756997426 - perf/throughput:3357.2875517509274 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:32 - global_seqlen/min:155062.0 - global_seqlen/max:155062.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:155062.0 - global_seqlen/balanced_max:155062.0 - global_seqlen/mean:155062.0 - actor/entropy:0.15179917216300964 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9660054445266724 - training/rollout_probs_diff_mean:0.004764268174767494 - training/rollout_probs_diff_std:0.015471293590962887 - training/rollout_actor_probs_pearson_corr:0.9967013597488403 - rollout_corr/training_ppl:1.1728383302688599 - rollout_corr/training_log_ppl:0.1567063182592392 - rollout_corr/kl:0.001109359902329743 - rollout_corr/k3_kl:0.00131770980078727 - rollout_corr/rollout_ppl:1.1714565753936768 - rollout_corr/rollout_log_ppl:0.15557187795639038 - rollout_corr/log_ppl_diff:0.00113444693852216 - rollout_corr/log_ppl_abs_diff:0.0033428347669541836 - rollout_corr/log_ppl_diff_max:0.03769339621067047 - rollout_corr/log_ppl_diff_min:-0.02666471153497696 - rollout_corr/ppl_ratio:1.0011459589004517 - rollout_corr/chi2_token:0.018025636672973633 - rollout_corr/chi2_seq:1.0667405128479004 - actor/pg_clipfrac:0.003429178022391473 - actor/ppo_kl:-0.00019621383260452302 - actor/pg_clipfrac_lower:1.53511288469114e-05 - actor/pg_loss:-0.0009466019982937723 - actor/loss:-0.0009466009214520454 - actor/grad_norm:1.6728515625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.00997257232666 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:32 - training/epoch:0 - critic/score/mean:0.3681640625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3681640625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.01058272272348404 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.01058272272348404 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:119.068359375 - response_length/max:128.0 - response_length/min:46.0 - response_length/clip_ratio:0.68359375 - response_length_non_aborted/mean:119.068359375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:46.0 - response_length_non_aborted/clip_ratio:0.68359375 - response/aborted_ratio:0.0 - prompt_length/mean:32.359375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.188718825003889 - timing_s/old_log_prob:8.461023649004346 - timing_s/adv:0.18581278300553095 - timing_s/update_actor:27.357424240995897 - timing_s/update_weights:1.8889294740001787 - timing_s/step:46.251449115006835 - timing_per_token_ms/adv:0.001198312823293463 - timing_per_token_ms/update_actor:0.17642893965636905 - timing_per_token_ms/gen:0.06716138333910641 - perf/total_num_tokens:155062 - perf/time_per_step:46.251449115006835 - perf/throughput:3352.5868479153937 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 24%|██▎ | 33/140 [26:57<1:25:13, 47.79s/it] (TaskRunnerV1 pid=2351) Training Progress: 24%|██▍ | 34/140 [27:43<1:23:41, 47.38s/it] (TaskRunnerV1 pid=2351) Training Progress: 25%|██▌ | 35/140 [28:39<1:26:57, 49.69s/it] (TaskRunnerV1 pid=2351) step:33 - global_seqlen/min:158543.0 - global_seqlen/max:158543.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:158543.0 - global_seqlen/balanced_max:158543.0 - global_seqlen/mean:158543.0 - actor/entropy:0.15634961426258087 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9818663001060486 - training/rollout_probs_diff_mean:0.004909126553684473 - training/rollout_probs_diff_std:0.016288381069898605 - training/rollout_actor_probs_pearson_corr:0.9965681433677673 - rollout_corr/training_ppl:1.1770024299621582 - rollout_corr/training_log_ppl:0.16038298606872559 - rollout_corr/kl:0.001130076590925455 - rollout_corr/k3_kl:0.0011229550000280142 - rollout_corr/rollout_ppl:1.1756765842437744 - rollout_corr/rollout_log_ppl:0.15930214524269104 - rollout_corr/log_ppl_diff:0.0010808381484821439 - rollout_corr/log_ppl_abs_diff:0.003478843020275235 - rollout_corr/log_ppl_diff_max:0.11174792051315308 - rollout_corr/log_ppl_diff_min:-0.015821203589439392 - rollout_corr/ppl_ratio:1.001098394393921 - rollout_corr/chi2_token:0.001912236213684082 - rollout_corr/chi2_seq:0.284237265586853 - actor/pg_clipfrac:0.003211183010231859 - actor/ppo_kl:-0.00012978875224689546 - actor/pg_clipfrac_lower:1.5148760818798715e-05 - actor/pg_loss:1.5251156355589046e-05 - actor/loss:1.5251338481903076e-05 - actor/grad_norm:1.6455078125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.0087685585022 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:33 - training/epoch:0 - critic/score/mean:0.2978515625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.2978515625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004186909180134535 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004186909180134535 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:121.5693359375 - response_length/max:128.0 - response_length/min:57.0 - response_length/clip_ratio:0.7451171875 - response_length_non_aborted/mean:121.5693359375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:57.0 - response_length_non_aborted/clip_ratio:0.7451171875 - response/aborted_ratio:0.0 - prompt_length/mean:33.2578125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.18778357499832 - timing_s/old_log_prob:8.673936159000732 - timing_s/adv:0.1817505489962059 - timing_s/update_actor:27.606427524995524 - timing_s/update_weights:1.8577356599998893 - timing_s/step:46.677365528994414 - timing_per_token_ms/adv:0.0011463801555174678 - timing_per_token_ms/update_actor:0.17412580514431747 - timing_per_token_ms/gen:0.06577219769934467 - perf/total_num_tokens:158543 - perf/time_per_step:46.677365528994414 - perf/throughput:3396.5712975278866 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:34 - global_seqlen/min:156309.0 - global_seqlen/max:156309.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:156309.0 - global_seqlen/balanced_max:156309.0 - global_seqlen/mean:156309.0 - actor/entropy:0.15395018458366394 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9540956020355225 - training/rollout_probs_diff_mean:0.004806154873222113 - training/rollout_probs_diff_std:0.015729669481515884 - training/rollout_actor_probs_pearson_corr:0.9968876242637634 - rollout_corr/training_ppl:1.1766655445098877 - rollout_corr/training_log_ppl:0.15978601574897766 - rollout_corr/kl:0.0010987339774146676 - rollout_corr/k3_kl:0.0010248764883726835 - rollout_corr/rollout_ppl:1.1753531694412231 - rollout_corr/rollout_log_ppl:0.15870030224323273 - rollout_corr/log_ppl_diff:0.0010857202578336 - rollout_corr/log_ppl_abs_diff:0.003303434234112501 - rollout_corr/log_ppl_diff_max:0.042020782828330994 - rollout_corr/log_ppl_diff_min:-0.019713252782821655 - rollout_corr/ppl_ratio:1.001096487045288 - rollout_corr/chi2_token:0.0019023418426513672 - rollout_corr/chi2_seq:0.2311471700668335 - actor/pg_clipfrac:0.0029708894053328554 - actor/ppo_kl:-8.063752284215298e-05 - actor/pg_clipfrac_lower:7.57025190978311e-06 - actor/pg_loss:0.0026566645537968725 - actor/loss:0.0026566656306385994 - actor/grad_norm:1.46240234375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.01045083999634 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:34 - training/epoch:0 - critic/score/mean:0.341796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.341796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.011054492555558681 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.011054492555558681 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:119.5751953125 - response_length/max:128.0 - response_length/min:48.0 - response_length/clip_ratio:0.6865234375 - response_length_non_aborted/mean:119.5751953125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:48.0 - response_length_non_aborted/clip_ratio:0.6865234375 - response/aborted_ratio:0.0 - prompt_length/mean:33.0703125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.199803331001021 - timing_s/old_log_prob:8.782577407000645 - timing_s/adv:0.18185719499888364 - timing_s/update_actor:27.06037784199725 - timing_s/update_weights:1.8614737509997212 - timing_s/step:46.26114247499936 - timing_per_token_ms/adv:0.0011634467305074157 - timing_per_token_ms/update_actor:0.173121047681178 - timing_per_token_ms/gen:0.06696723697170992 - perf/total_num_tokens:156309 - perf/time_per_step:46.26114247499936 - perf/throughput:3378.840029393419 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:35 - global_seqlen/min:153071.0 - global_seqlen/max:153071.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:153071.0 - global_seqlen/balanced_max:153071.0 - global_seqlen/mean:153071.0 - actor/entropy:0.16247770190238953 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8045393228530884 - training/rollout_probs_diff_mean:0.005153025034815073 - training/rollout_probs_diff_std:0.0160475242882967 - training/rollout_actor_probs_pearson_corr:0.9966152310371399 - rollout_corr/training_ppl:1.181096076965332 - rollout_corr/training_log_ppl:0.16376367211341858 - rollout_corr/kl:0.0010738915298134089 - rollout_corr/k3_kl:0.0010303262388333678 - rollout_corr/rollout_ppl:1.179750919342041 - rollout_corr/rollout_log_ppl:0.16266566514968872 - rollout_corr/log_ppl_diff:0.0010980131337419152 - rollout_corr/log_ppl_abs_diff:0.003394374158233404 - rollout_corr/log_ppl_diff_max:0.02942393720149994 - rollout_corr/log_ppl_diff_min:-0.014693722128868103 - rollout_corr/ppl_ratio:1.0011087656021118 - rollout_corr/chi2_token:0.0018875598907470703 - rollout_corr/chi2_seq:0.2167888879776001 - actor/pg_clipfrac:0.0035839220945490524 - actor/ppo_kl:-6.763631023432028e-05 - actor/pg_clipfrac_lower:3.447715668395782e-05 - actor/pg_loss:0.000137659051688388 - actor/loss:0.00013765692710876465 - actor/grad_norm:1.9228515625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.10497713088989 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.385) - val-core/char_count/acc/mean@1:np.float64(0.385) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:35 - training/epoch:0 - critic/score/mean:0.3935546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3935546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.01419057510793209 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.01419057510793209 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:117.0771484375 - response_length/max:128.0 - response_length/min:36.0 - response_length/clip_ratio:0.6298828125 - response_length_non_aborted/mean:117.0771484375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:36.0 - response_length_non_aborted/clip_ratio:0.6298828125 - response/aborted_ratio:0.0 - prompt_length/mean:32.40625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.204253285999584 - timing_s/old_log_prob:8.551306348999788 - timing_s/adv:0.18434670499846106 - timing_s/update_actor:27.124285865000274 - timing_s/update_weights:1.8498662860001787 - timing_s/step:46.081266563996905 - timing_s/testing:8.645989310003642 - timing_per_token_ms/adv:0.001204321556653194 - timing_per_token_ms/update_actor:0.17720068376766515 - timing_per_token_ms/gen:0.0684332186642387 - perf/total_num_tokens:153071 - perf/time_per_step:46.081266563996905 - perf/throughput:3321.76199600368 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 26%|██▌ | 36/140 [29:25<1:24:28, 48.74s/it] (TransferQueueController pid=2438) 2026-11-01 17:45:00,081 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 35971, Total req/min: 7194.13, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.00 minutes; (TransferQueueController pid=2438) Per-operation statistics: SET_CUSTOM_META: req_count=13844, req/min=2768.77, avg_time=0.000009s, max_time=0.000129s, min_time=0.000003s; KV_RETRIEVE_META: req_count=13890, req/min=2777.97, avg_time=0.000221s, max_time=0.067365s, min_time=0.000062s; KV_LIST: req_count=42, req/min=8.40, avg_time=0.000228s, max_time=0.000667s, min_time=0.000028s; NOTIFY_DATA_UPDATE: req_count=8172, req/min=1634.38, avg_time=0.000079s, max_time=0.010218s, min_time=0.000030s; CLEAR_META: req_count=17, req/min=3.40, avg_time=0.001598s, max_time=0.002076s, min_time=0.000826s; KV_RETRIEVE_KEYS: req_count=6, req/min=1.20, avg_time=0.000112s, max_time=0.000120s, min_time=0.000105s (SimpleStorageUnit pid=2599) Per-operation statistics: PUT_DATA: req_count=1055, req/min=210.96, avg_time=0.000601s, max_time=0.001433s, min_time=0.000307s; CLEAR_DATA: req_count=17, req/min=3.40, avg_time=0.001402s, max_time=0.003200s, min_time=0.000393s; GET_DATA: req_count=35, req/min=7.00, avg_time=0.000604s, max_time=0.001227s, min_time=0.000385s (TaskRunnerV1 pid=2351) Training Progress: 26%|██▋ | 37/140 [30:12<1:22:30, 48.07s/it] (SimpleStorageUnit pid=2582) 2026-11-01 17:45:04,956 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_e1220d54: [Performance] Total success requests: 1233, Total req/min: 245.85, Total avg process time: 0.0006s; [repeated 8x across cluster] (SimpleStorageUnit pid=2582) Time range: last 5.02 minutes; [repeated 8x across cluster] (SimpleStorageUnit pid=2582) Per-operation statistics: PUT_DATA: req_count=1180, req/min=235.29, avg_time=0.000603s, max_time=0.003487s, min_time=0.000314s; CLEAR_DATA: req_count=18, req/min=3.59, avg_time=0.001277s, max_time=0.003290s, min_time=0.000412s; GET_DATA: req_count=35, req/min=6.98, avg_time=0.000595s, max_time=0.001234s, min_time=0.000411s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 27%|██▋ | 38/140 [30:57<1:20:36, 47.42s/it] (TaskRunnerV1 pid=2351) step:36 - global_seqlen/min:157716.0 - global_seqlen/max:157716.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:157716.0 - global_seqlen/balanced_max:157716.0 - global_seqlen/mean:157716.0 - actor/entropy:0.1500566154718399 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9313358664512634 - training/rollout_probs_diff_mean:0.004732903558760881 - training/rollout_probs_diff_std:0.015804219990968704 - training/rollout_actor_probs_pearson_corr:0.9966511726379395 - rollout_corr/training_ppl:1.170202612876892 - rollout_corr/training_log_ppl:0.1540813148021698 - rollout_corr/kl:0.001097743515856564 - rollout_corr/k3_kl:0.0009988612728193402 - rollout_corr/rollout_ppl:1.1688053607940674 - rollout_corr/rollout_log_ppl:0.15295106172561646 - rollout_corr/log_ppl_diff:0.0011302597122266889 - rollout_corr/log_ppl_abs_diff:0.003236727323383093 - rollout_corr/log_ppl_diff_max:0.04249155521392822 - rollout_corr/log_ppl_diff_min:-0.015820175409317017 - rollout_corr/ppl_ratio:1.0011405944824219 - rollout_corr/chi2_token:0.001771688461303711 - rollout_corr/chi2_seq:0.23608613014221191 - actor/pg_clipfrac:0.0030149092296293625 - actor/ppo_kl:1.5152294395193177e-05 - actor/pg_clipfrac_lower:1.5327695412755322e-05 - actor/pg_loss:-0.0019047515738748189 - actor/loss:-0.0019047502428293228 - actor/grad_norm:1.53515625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.088064670562744 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:36 - training/epoch:0 - critic/score/mean:0.32421875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.32421875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005307271145284176 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005307271145284176 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:120.51171875 - response_length/max:128.0 - response_length/min:27.0 - response_length/clip_ratio:0.7138671875 - response_length_non_aborted/mean:120.51171875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:27.0 - response_length_non_aborted/clip_ratio:0.7138671875 - response/aborted_ratio:0.0 - prompt_length/mean:33.5078125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:8.196738619000826 - timing_s/old_log_prob:8.574247743999877 - timing_s/adv:0.1856009469993296 - timing_s/update_actor:27.416065974000958 - timing_s/update_weights:1.8477055030016345 - timing_s/step:46.38889875099994 - timing_per_token_ms/adv:0.0011768048073710316 - timing_per_token_ms/update_actor:0.17383186216998248 - timing_per_token_ms/gen:0.06642198485463052 - perf/total_num_tokens:157716 - perf/time_per_step:46.38889875099994 - perf/throughput:3399.8651454643623 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:37 - global_seqlen/min:154838.0 - global_seqlen/max:154838.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:154838.0 - global_seqlen/balanced_max:154838.0 - global_seqlen/mean:154838.0 - actor/entropy:0.15471495687961578 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8908341526985168 - training/rollout_probs_diff_mean:0.004772553686052561 - training/rollout_probs_diff_std:0.015160957351326942 - training/rollout_actor_probs_pearson_corr:0.9970572590827942 - rollout_corr/training_ppl:1.1780458688735962 - rollout_corr/training_log_ppl:0.1606195569038391 - rollout_corr/kl:0.0009391125058755279 - rollout_corr/k3_kl:0.0009624582598917186 - rollout_corr/rollout_ppl:1.1768778562545776 - rollout_corr/rollout_log_ppl:0.15967832505702972 - rollout_corr/log_ppl_diff:0.0009412241633981466 - rollout_corr/log_ppl_abs_diff:0.003301330842077732 - rollout_corr/log_ppl_diff_max:0.029013358056545258 - rollout_corr/log_ppl_diff_min:-0.010484971106052399 - rollout_corr/ppl_ratio:1.0009514093399048 - rollout_corr/chi2_token:0.0018807649612426758 - rollout_corr/chi2_seq:0.21838974952697754 - actor/pg_clipfrac:0.0034064328307673955 - actor/ppo_kl:0.000295048036832668 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0017858350111055188 - actor/loss:-0.0017858351347967982 - actor/grad_norm:1.80859375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.107948303222656 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:37 - training/epoch:0 - critic/score/mean:0.3408203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3408203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0023696136195212603 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0023696136195212603 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:118.529296875 - response_length/max:128.0 - response_length/min:39.0 - response_length/clip_ratio:0.662109375 - response_length_non_aborted/mean:118.529296875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:39.0 - response_length_non_aborted/clip_ratio:0.662109375 - response/aborted_ratio:0.0 - prompt_length/mean:32.6796875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.182004852002137 - timing_s/old_log_prob:8.67764714799705 - timing_s/adv:0.18420762999448925 - timing_s/update_actor:27.29906567100261 - timing_s/update_weights:1.8658806719977292 - timing_s/step:46.37958914299816 - timing_per_token_ms/adv:0.0011896797297465045 - timing_per_token_ms/update_actor:0.17630727386689707 - timing_per_token_ms/gen:0.067411511954802 - perf/total_num_tokens:154838 - perf/time_per_step:46.37958914299816 - perf/throughput:3338.494429577663 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:38 - global_seqlen/min:151569.0 - global_seqlen/max:151569.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151569.0 - global_seqlen/balanced_max:151569.0 - global_seqlen/mean:151569.0 - actor/entropy:0.15524037182331085 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9494014382362366 - training/rollout_probs_diff_mean:0.00497736781835556 - training/rollout_probs_diff_std:0.01642904430627823 - training/rollout_actor_probs_pearson_corr:0.9964492321014404 - rollout_corr/training_ppl:1.1794257164001465 - rollout_corr/training_log_ppl:0.1623198390007019 - rollout_corr/kl:0.0013736766995862126 - rollout_corr/k3_kl:0.0011613890528678894 - rollout_corr/rollout_ppl:1.177813172340393 - rollout_corr/rollout_log_ppl:0.1610049456357956 - rollout_corr/log_ppl_diff:0.001314897439442575 - rollout_corr/log_ppl_abs_diff:0.003553782356902957 - rollout_corr/log_ppl_diff_max:0.04415535181760788 - rollout_corr/log_ppl_diff_min:-0.01657959818840027 - rollout_corr/ppl_ratio:1.0013281106948853 - rollout_corr/chi2_token:0.001734614372253418 - rollout_corr/chi2_seq:0.14154648780822754 - actor/pg_clipfrac:0.0033960604632738973 - actor/ppo_kl:-0.00015172818734754404 - actor/pg_clipfrac_lower:1.5141879581885101e-05 - actor/pg_loss:0.0013558040373027325 - actor/loss:0.0013558031059801579 - actor/grad_norm:1.8017578125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.06143760681152 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:38 - training/epoch:0 - critic/score/mean:0.396484375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.396484375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008364254608750343 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008364254608750343 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.4697265625 - response_length/max:128.0 - response_length/min:35.0 - response_length/clip_ratio:0.6103515625 - response_length_non_aborted/mean:115.4697265625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:35.0 - response_length_non_aborted/clip_ratio:0.6103515625 - response/aborted_ratio:0.0 - prompt_length/mean:32.546875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:8.1962768279991 - timing_s/old_log_prob:8.555258088999835 - timing_s/adv:0.18617968099715654 - timing_s/update_actor:26.81055398299941 - timing_s/update_weights:1.8513770489953458 - timing_s/step:45.77432909700292 - timing_per_token_ms/adv:0.0012283493392260723 - timing_per_token_ms/update_actor:0.17688679072237337 - timing_per_token_ms/gen:0.06931839910013532 - perf/total_num_tokens:151569 - perf/time_per_step:45.77432909700292 - perf/throughput:3311.222752796698 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 28%|██▊ | 39/140 [31:44<1:19:16, 47.09s/it] (TaskRunnerV1 pid=2351) Training Progress: 29%|██▊ | 40/140 [32:39<1:22:24, 49.45s/it] (TaskRunnerV1 pid=2351) Training Progress: 29%|██▉ | 41/140 [33:25<1:20:01, 48.50s/it] (TaskRunnerV1 pid=2351) step:39 - global_seqlen/min:154084.0 - global_seqlen/max:154084.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:154084.0 - global_seqlen/balanced_max:154084.0 - global_seqlen/mean:154084.0 - actor/entropy:0.15167327225208282 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8667334914207458 - training/rollout_probs_diff_mean:0.004778655245900154 - training/rollout_probs_diff_std:0.016317807137966156 - training/rollout_actor_probs_pearson_corr:0.996464729309082 - rollout_corr/training_ppl:1.176523208618164 - rollout_corr/training_log_ppl:0.1594710648059845 - rollout_corr/kl:0.0012108446098864079 - rollout_corr/k3_kl:0.0013877643505111337 - rollout_corr/rollout_ppl:1.175079107284546 - rollout_corr/rollout_log_ppl:0.15830901265144348 - rollout_corr/log_ppl_diff:0.0011620396981015801 - rollout_corr/log_ppl_abs_diff:0.0036152524407953024 - rollout_corr/log_ppl_diff_max:0.06651103496551514 - rollout_corr/log_ppl_diff_min:-0.020473748445510864 - rollout_corr/ppl_ratio:1.001179814338684 - rollout_corr/chi2_token:0.007096529006958008 - rollout_corr/chi2_seq:0.5327091217041016 - actor/pg_clipfrac:0.003234941022431788 - actor/ppo_kl:-0.00026864041819862905 - actor/pg_clipfrac_lower:1.552121527007936e-05 - actor/pg_loss:0.0019714974041562527 - actor/loss:0.001971498131752014 - actor/grad_norm:1.6162109375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.053295612335205 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:39 - training/epoch:0 - critic/score/mean:0.3388671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3388671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00974367093294859 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.00974367093294859 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:117.35546875 - response_length/max:128.0 - response_length/min:36.0 - response_length/clip_ratio:0.6826171875 - response_length_non_aborted/mean:117.35546875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:36.0 - response_length_non_aborted/clip_ratio:0.6826171875 - response/aborted_ratio:0.0 - prompt_length/mean:33.1171875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:8.188125029999355 - timing_s/old_log_prob:8.37517212700186 - timing_s/adv:0.38320123099401826 - timing_s/update_actor:27.17515648999688 - timing_s/update_weights:1.8633571119935368 - timing_s/step:46.15144307100127 - timing_per_token_ms/adv:0.0024869631564212913 - timing_per_token_ms/update_actor:0.176365855572265 - timing_per_token_ms/gen:0.06813671262856036 - perf/total_num_tokens:154084 - perf/time_per_step:46.15144307100127 - perf/throughput:3338.660500018404 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:40 - global_seqlen/min:154018.0 - global_seqlen/max:154018.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:154018.0 - global_seqlen/balanced_max:154018.0 - global_seqlen/mean:154018.0 - actor/entropy:0.1487022042274475 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8287205100059509 - training/rollout_probs_diff_mean:0.004741809796541929 - training/rollout_probs_diff_std:0.015793094411492348 - training/rollout_actor_probs_pearson_corr:0.9965951442718506 - rollout_corr/training_ppl:1.168735384941101 - rollout_corr/training_log_ppl:0.15344664454460144 - rollout_corr/kl:0.0008335101301781833 - rollout_corr/k3_kl:0.0010526381665840745 - rollout_corr/rollout_ppl:1.1677144765853882 - rollout_corr/rollout_log_ppl:0.1526038646697998 - rollout_corr/log_ppl_diff:0.0008427916327491403 - rollout_corr/log_ppl_abs_diff:0.003400250105187297 - rollout_corr/log_ppl_diff_max:0.0581321120262146 - rollout_corr/log_ppl_diff_min:-0.015006691217422485 - rollout_corr/ppl_ratio:1.0008563995361328 - rollout_corr/chi2_token:0.0023703575134277344 - rollout_corr/chi2_seq:0.33859872817993164 - actor/pg_clipfrac:0.003474076496786438 - actor/ppo_kl:-4.892750932109863e-06 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0006947676884010434 - actor/loss:0.0006947675719857216 - actor/grad_norm:1.650390625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:41.99624729156494 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.396) - val-core/char_count/acc/mean@1:np.float64(0.396) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:40 - training/epoch:0 - critic/score/mean:0.3818359375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3818359375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.012200691737234592 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.012200691737234592 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:117.720703125 - response_length/max:128.0 - response_length/min:29.0 - response_length/clip_ratio:0.6474609375 - response_length_non_aborted/mean:117.720703125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:29.0 - response_length_non_aborted/clip_ratio:0.6474609375 - response/aborted_ratio:0.0 - prompt_length/mean:32.6875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.19158697000239 - timing_s/old_log_prob:8.440896763000637 - timing_s/adv:0.38419301699468633 - timing_s/update_actor:27.141284571996948 - timing_s/update_weights:1.8701918270016904 - timing_s/step:46.1940748440029 - timing_s/testing:8.6292545910037 - timing_per_token_ms/adv:0.002494468289386217 - timing_per_token_ms/update_actor:0.17622151029098512 - timing_per_token_ms/gen:0.06795403389579406 - perf/total_num_tokens:154018 - perf/time_per_step:46.1940748440029 - perf/throughput:3334.1505489636456 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:41 - global_seqlen/min:152391.0 - global_seqlen/max:152391.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:152391.0 - global_seqlen/balanced_max:152391.0 - global_seqlen/mean:152391.0 - actor/entropy:0.15160557627677917 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9398549795150757 - training/rollout_probs_diff_mean:0.004851170349866152 - training/rollout_probs_diff_std:0.01623649150133133 - training/rollout_actor_probs_pearson_corr:0.9964455962181091 - rollout_corr/training_ppl:1.1733283996582031 - rollout_corr/training_log_ppl:0.1569269299507141 - rollout_corr/kl:0.0011366346152499318 - rollout_corr/k3_kl:0.001110088312998414 - rollout_corr/rollout_ppl:1.17194402217865 - rollout_corr/rollout_log_ppl:0.1557861715555191 - rollout_corr/log_ppl_diff:0.0011407771380618215 - rollout_corr/log_ppl_abs_diff:0.0034612370654940605 - rollout_corr/log_ppl_diff_max:0.05315917730331421 - rollout_corr/log_ppl_diff_min:-0.015865415334701538 - rollout_corr/ppl_ratio:1.0011533498764038 - rollout_corr/chi2_token:0.0020112991333007812 - rollout_corr/chi2_seq:0.20966625213623047 - actor/pg_clipfrac:0.003224991713068448 - actor/ppo_kl:2.0760582335318155e-05 - actor/pg_clipfrac_lower:4.01861058586898e-05 - actor/pg_loss:0.001681575567999971 - actor/loss:0.0016815769486129284 - actor/grad_norm:1.7802734375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.07521677017212 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:41 - training/epoch:0 - critic/score/mean:0.4072265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4072265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.010489648208022118 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.010489648208022118 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:116.4521484375 - response_length/max:128.0 - response_length/min:39.0 - response_length/clip_ratio:0.603515625 - response_length_non_aborted/mean:116.4521484375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:39.0 - response_length_non_aborted/clip_ratio:0.603515625 - response/aborted_ratio:0.0 - prompt_length/mean:32.3671875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.22084515899769 - timing_s/old_log_prob:8.614742768993892 - timing_s/adv:0.25337462100287667 - timing_s/update_actor:26.99214805999509 - timing_s/update_weights:1.8506844570001704 - timing_s/step:46.10078556499502 - timing_per_token_ms/adv:0.001662661318600683 - timing_per_token_ms/update_actor:0.17712429251068035 - timing_per_token_ms/gen:0.06893963922780187 - perf/total_num_tokens:152391 - perf/time_per_step:46.10078556499502 - perf/throughput:3305.605276186717 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 30%|███ | 42/140 [34:11<1:18:08, 47.84s/it] (TransferQueueController pid=2438) 2026-11-01 17:50:00,164 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 31981, Total req/min: 6394.43, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.00 minutes; (TransferQueueController pid=2438) Per-operation statistics: SET_CUSTOM_META: req_count=11850, req/min=2369.34, avg_time=0.000009s, max_time=0.000153s, min_time=0.000003s; KV_RETRIEVE_META: req_count=11893, req/min=2377.94, avg_time=0.000248s, max_time=0.020749s, min_time=0.000065s; NOTIFY_DATA_UPDATE: req_count=8178, req/min=1635.15, avg_time=0.000079s, max_time=0.001061s, min_time=0.000031s; KV_LIST: req_count=38, req/min=7.60, avg_time=0.000233s, max_time=0.000679s, min_time=0.000038s; CLEAR_META: req_count=15, req/min=3.00, avg_time=0.001753s, max_time=0.002155s, min_time=0.000969s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.40, avg_time=0.000114s, max_time=0.000120s, min_time=0.000107s (SimpleStorageUnit pid=2599) Per-operation statistics: PUT_DATA: req_count=985, req/min=181.35, avg_time=0.000580s, max_time=0.001260s, min_time=0.000313s; CLEAR_DATA: req_count=15, req/min=2.76, avg_time=0.001309s, max_time=0.004429s, min_time=0.000414s; GET_DATA: req_count=37, req/min=6.81, avg_time=0.000604s, max_time=0.001145s, min_time=0.000378s (SimpleStorageUnit pid=2599) 2026-11-01 17:50:28,986 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1037, Total req/min: 190.93, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.43 minutes; (SimpleStorageUnit pid=2596) 2026-11-01 17:50:28,986 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_72b4f821: [Performance] Total success requests: 954, Total req/min: 176.65, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2596) Time range: last 5.40 minutes; (SimpleStorageUnit pid=2596) Per-operation statistics: GET_DATA: req_count=37, req/min=6.85, avg_time=0.000611s, max_time=0.001226s, min_time=0.000402s; PUT_DATA: req_count=903, req/min=167.21, avg_time=0.000600s, max_time=0.001696s, min_time=0.000311s; CLEAR_DATA: req_count=14, req/min=2.59, avg_time=0.001503s, max_time=0.004066s, min_time=0.000437s (TaskRunnerV1 pid=2351) Training Progress: 31%|███ | 43/140 [34:57<1:16:24, 47.26s/it] (TaskRunnerV1 pid=2351) Training Progress: 31%|███▏ | 44/140 [35:44<1:15:12, 47.00s/it] (SimpleStorageUnit pid=2583) Per-operation statistics: PUT_DATA: req_count=961, req/min=177.20, avg_time=0.000580s, max_time=0.001818s, min_time=0.000316s; CLEAR_DATA: req_count=15, req/min=2.77, avg_time=0.001324s, max_time=0.003816s, min_time=0.000414s; GET_DATA: req_count=37, req/min=6.82, avg_time=0.000611s, max_time=0.001095s, min_time=0.000421s [repeated 5x across cluster] (SimpleStorageUnit pid=2583) 2026-11-01 17:50:28,986 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1013, Total req/min: 186.79, Total avg process time: 0.0006s; [repeated 6x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.42 minutes; [repeated 6x across cluster] (SimpleStorageUnit pid=2582) Per-operation statistics: GET_DATA: req_count=37, req/min=6.85, avg_time=0.000620s, max_time=0.001184s, min_time=0.000387s; PUT_DATA: req_count=903, req/min=167.21, avg_time=0.000579s, max_time=0.001621s, min_time=0.000318s; CLEAR_DATA: req_count=14, req/min=2.59, avg_time=0.001696s, max_time=0.004238s, min_time=0.000417s (TaskRunnerV1 pid=2351) step:42 - global_seqlen/min:154053.0 - global_seqlen/max:154053.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:154053.0 - global_seqlen/balanced_max:154053.0 - global_seqlen/mean:154053.0 - actor/entropy:0.1483476459980011 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7723837494850159 - training/rollout_probs_diff_mean:0.004622644744813442 - training/rollout_probs_diff_std:0.015028214082121849 - training/rollout_actor_probs_pearson_corr:0.9970051050186157 - rollout_corr/training_ppl:1.1689871549606323 - rollout_corr/training_log_ppl:0.15352186560630798 - rollout_corr/kl:0.0010418308665975928 - rollout_corr/k3_kl:0.0010209353640675545 - rollout_corr/rollout_ppl:1.1676576137542725 - rollout_corr/rollout_log_ppl:0.1524287760257721 - rollout_corr/log_ppl_diff:0.001093104830943048 - rollout_corr/log_ppl_abs_diff:0.00329991290345788 - rollout_corr/log_ppl_diff_max:0.044341638684272766 - rollout_corr/log_ppl_diff_min:-0.018483132123947144 - rollout_corr/ppl_ratio:1.001104474067688 - rollout_corr/chi2_token:0.002234339714050293 - rollout_corr/chi2_seq:0.25338590145111084 - actor/pg_clipfrac:0.0030376529175555333 - actor/ppo_kl:5.093863637739558e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0006590872544620652 - actor/loss:0.0006590881384909153 - actor/grad_norm:1.681640625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.15001964569092 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:42 - training/epoch:0 - critic/score/mean:0.390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.009559937752783298 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.009559937752783298 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:117.7861328125 - response_length/max:128.0 - response_length/min:47.0 - response_length/clip_ratio:0.6591796875 - response_length_non_aborted/mean:117.7861328125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:47.0 - response_length_non_aborted/clip_ratio:0.6591796875 - response/aborted_ratio:0.0 - prompt_length/mean:32.65625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.211838039002032 - timing_s/old_log_prob:8.579985566997493 - timing_s/adv:0.1862618390005082 - timing_s/update_actor:27.15500401800091 - timing_s/update_weights:1.8705050019998453 - timing_s/step:46.182251833997725 - timing_per_token_ms/adv:0.0012090763503502574 - timing_per_token_ms/update_actor:0.17627053038889803 - timing_per_token_ms/gen:0.06808418693674838 - perf/total_num_tokens:154053 - perf/time_per_step:46.182251833997725 - perf/throughput:3335.7619839271606 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:43 - global_seqlen/min:152558.0 - global_seqlen/max:152558.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:152558.0 - global_seqlen/balanced_max:152558.0 - global_seqlen/mean:152558.0 - actor/entropy:0.14895586669445038 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.70174640417099 - training/rollout_probs_diff_mean:0.00472778407856822 - training/rollout_probs_diff_std:0.015411971136927605 - training/rollout_actor_probs_pearson_corr:0.9968196749687195 - rollout_corr/training_ppl:1.1684328317642212 - rollout_corr/training_log_ppl:0.15276303887367249 - rollout_corr/kl:0.0007204193389043212 - rollout_corr/k3_kl:0.0009749169694259763 - rollout_corr/rollout_ppl:1.1675645112991333 - rollout_corr/rollout_log_ppl:0.15205658972263336 - rollout_corr/log_ppl_diff:0.000706447521224618 - rollout_corr/log_ppl_abs_diff:0.003311133710667491 - rollout_corr/log_ppl_diff_max:0.02573704719543457 - rollout_corr/log_ppl_diff_min:-0.019078880548477173 - rollout_corr/ppl_ratio:1.000717043876648 - rollout_corr/chi2_token:0.002467036247253418 - rollout_corr/chi2_seq:0.3114798069000244 - actor/pg_clipfrac:0.002968695574963931 - actor/ppo_kl:0.0005756321248971592 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0016460847982671112 - actor/loss:0.0016460842452943325 - actor/grad_norm:1.708984375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.05587673187256 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:43 - training/epoch:0 - critic/score/mean:0.3583984375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3583984375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.01596231758594513 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.01596231758594513 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:116.154296875 - response_length/max:128.0 - response_length/min:40.0 - response_length/clip_ratio:0.6259765625 - response_length_non_aborted/mean:116.154296875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:40.0 - response_length_non_aborted/clip_ratio:0.6259765625 - response/aborted_ratio:0.0 - prompt_length/mean:32.828125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.186620758002391 - timing_s/old_log_prob:8.341451825996046 - timing_s/adv:0.3808731700046337 - timing_s/update_actor:26.8472849030004 - timing_s/update_weights:1.8654365460024565 - timing_s/step:45.785384725997574 - timing_per_token_ms/adv:0.0024965794648896403 - timing_per_token_ms/update_actor:0.17598083943811796 - timing_per_token_ms/gen:0.06882867917138094 - perf/total_num_tokens:152558 - perf/time_per_step:45.785384725997574 - perf/throughput:3332.02398348256 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:44 - global_seqlen/min:154640.0 - global_seqlen/max:154640.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:154640.0 - global_seqlen/balanced_max:154640.0 - global_seqlen/mean:154640.0 - actor/entropy:0.1497325897216797 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8160721063613892 - training/rollout_probs_diff_mean:0.004719313699752092 - training/rollout_probs_diff_std:0.015162188559770584 - training/rollout_actor_probs_pearson_corr:0.9968194365501404 - rollout_corr/training_ppl:1.171676754951477 - rollout_corr/training_log_ppl:0.15568391978740692 - rollout_corr/kl:0.0009205284295603633 - rollout_corr/k3_kl:0.0009131142869591713 - rollout_corr/rollout_ppl:1.170450210571289 - rollout_corr/rollout_log_ppl:0.1546923816204071 - rollout_corr/log_ppl_diff:0.000991549575701356 - rollout_corr/log_ppl_abs_diff:0.0031638771761208773 - rollout_corr/log_ppl_diff_max:0.023549243807792664 - rollout_corr/log_ppl_diff_min:-0.020318984985351562 - rollout_corr/ppl_ratio:1.001001000404358 - rollout_corr/chi2_token:0.001900792121887207 - rollout_corr/chi2_seq:0.2936824560165405 - actor/pg_clipfrac:0.0033827057535139224 - actor/ppo_kl:0.00012676053917933436 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.00014896742140990682 - actor/loss:-0.00014896783977746964 - actor/grad_norm:1.62890625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.029512882232666 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:44 - training/epoch:0 - critic/score/mean:0.357421875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.357421875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005970145110040903 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005970145110040903 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:118.3046875 - response_length/max:128.0 - response_length/min:26.0 - response_length/clip_ratio:0.6484375 - response_length_non_aborted/mean:118.3046875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:26.0 - response_length_non_aborted/clip_ratio:0.6484375 - response/aborted_ratio:0.0 - prompt_length/mean:32.7109375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.187487663002685 - timing_s/old_log_prob:8.427109213000222 - timing_s/adv:0.38298682200547773 - timing_s/update_actor:27.22220263499912 - timing_s/update_weights:1.8788008789997548 - timing_s/step:46.27015743499942 - timing_per_token_ms/adv:0.0024766349069159192 - timing_per_token_ms/update_actor:0.17603597151447958 - timing_per_token_ms/gen:0.06758475585256128 - perf/total_num_tokens:154640 - perf/time_per_step:46.27015743499942 - perf/throughput:3342.110953852689 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 32%|███▏ | 45/140 [36:38<1:18:02, 49.29s/it] (TaskRunnerV1 pid=2351) Training Progress: 33%|███▎ | 46/140 [37:24<1:15:30, 48.20s/it] (TaskRunnerV1 pid=2351) Training Progress: 34%|███▎ | 47/140 [38:10<1:13:35, 47.48s/it] (TaskRunnerV1 pid=2351) step:45 - global_seqlen/min:152028.0 - global_seqlen/max:152028.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:152028.0 - global_seqlen/balanced_max:152028.0 - global_seqlen/mean:152028.0 - actor/entropy:0.14946819841861725 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9169308543205261 - training/rollout_probs_diff_mean:0.004865668248385191 - training/rollout_probs_diff_std:0.015750741586089134 - training/rollout_actor_probs_pearson_corr:0.9966789484024048 - rollout_corr/training_ppl:1.1700807809829712 - rollout_corr/training_log_ppl:0.15447914600372314 - rollout_corr/kl:0.0009982101619243622 - rollout_corr/k3_kl:0.0010264803422614932 - rollout_corr/rollout_ppl:1.1688787937164307 - rollout_corr/rollout_log_ppl:0.1535036265850067 - rollout_corr/log_ppl_diff:0.0009755092905834317 - rollout_corr/log_ppl_abs_diff:0.0034313350915908813 - rollout_corr/log_ppl_diff_max:0.04056914895772934 - rollout_corr/log_ppl_diff_min:-0.01720041036605835 - rollout_corr/ppl_ratio:1.0009874105453491 - rollout_corr/chi2_token:0.0019532442092895508 - rollout_corr/chi2_seq:0.22794854640960693 - actor/pg_clipfrac:0.0037245436784966537 - actor/ppo_kl:5.261397523762905e-05 - actor/pg_clipfrac_lower:9.71930664187918e-06 - actor/pg_loss:0.0013663832214660943 - actor/loss:0.0013663817662745714 - actor/grad_norm:1.837890625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.03270196914673 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.405) - val-core/char_count/acc/mean@1:np.float64(0.405) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:45 - training/epoch:0 - critic/score/mean:0.4013671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4013671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004751681350171566 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004751681350171566 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:116.18359375 - response_length/max:128.0 - response_length/min:41.0 - response_length/clip_ratio:0.619140625 - response_length_non_aborted/mean:116.18359375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:41.0 - response_length_non_aborted/clip_ratio:0.619140625 - response/aborted_ratio:0.0 - prompt_length/mean:32.28125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:8.189536773003056 - timing_s/old_log_prob:8.309649971000908 - timing_s/adv:0.19913425499544246 - timing_s/update_actor:27.102617127005942 - timing_s/update_weights:1.8555758240036084 - timing_s/step:45.827686550997896 - timing_s/testing:8.605721092004387 - timing_per_token_ms/adv:0.0013098524942473917 - timing_per_token_ms/update_actor:0.17827385170498816 - timing_per_token_ms/gen:0.06883583341461062 - perf/total_num_tokens:152028 - perf/time_per_step:45.827686550997896 - perf/throughput:3317.3832554436376 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:46 - global_seqlen/min:150544.0 - global_seqlen/max:150544.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:150544.0 - global_seqlen/balanced_max:150544.0 - global_seqlen/mean:150544.0 - actor/entropy:0.1590641736984253 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9976609945297241 - training/rollout_probs_diff_mean:0.005161783192306757 - training/rollout_probs_diff_std:0.017834605649113655 - training/rollout_actor_probs_pearson_corr:0.9959326386451721 - rollout_corr/training_ppl:1.1844581365585327 - rollout_corr/training_log_ppl:0.16627559065818787 - rollout_corr/kl:0.0011528597678989172 - rollout_corr/k3_kl:0.0012805259320884943 - rollout_corr/rollout_ppl:1.1830570697784424 - rollout_corr/rollout_log_ppl:0.16512303054332733 - rollout_corr/log_ppl_diff:0.0011525636073201895 - rollout_corr/log_ppl_abs_diff:0.003725567599758506 - rollout_corr/log_ppl_diff_max:0.05143319070339203 - rollout_corr/log_ppl_diff_min:-0.022520720958709717 - rollout_corr/ppl_ratio:1.0011671781539917 - rollout_corr/chi2_token:0.0038862228393554688 - rollout_corr/chi2_seq:0.7306045293807983 - actor/pg_clipfrac:0.0035461922864972926 - actor/ppo_kl:0.00025704152252122486 - actor/pg_clipfrac_lower:3.8202542782528326e-05 - actor/pg_loss:0.00010839919559657574 - actor/loss:0.00010839937021955848 - actor/grad_norm:1.7666015625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.0550012588501 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:46 - training/epoch:0 - critic/score/mean:0.3671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008120015263557434 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008120015263557434 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:114.7890625 - response_length/max:128.0 - response_length/min:42.0 - response_length/clip_ratio:0.5966796875 - response_length_non_aborted/mean:114.7890625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:42.0 - response_length_non_aborted/clip_ratio:0.5966796875 - response/aborted_ratio:0.0 - prompt_length/mean:32.2265625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.199581508997653 - timing_s/old_log_prob:8.476946337003028 - timing_s/adv:0.18960092299676035 - timing_s/update_actor:26.621618967998074 - timing_s/update_weights:1.8526324710037443 - timing_s/step:45.5253181060034 - timing_per_token_ms/adv:0.0012594385893609865 - timing_per_token_ms/update_actor:0.17683613407374638 - timing_per_token_ms/gen:0.06975755044066607 - perf/total_num_tokens:150544 - perf/time_per_step:45.5253181060034 - perf/throughput:3306.8192878842915 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:47 - global_seqlen/min:151252.0 - global_seqlen/max:151252.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151252.0 - global_seqlen/balanced_max:151252.0 - global_seqlen/mean:151252.0 - actor/entropy:0.14814257621765137 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5248488187789917 - training/rollout_probs_diff_mean:0.004699824843555689 - training/rollout_probs_diff_std:0.015178998932242393 - training/rollout_actor_probs_pearson_corr:0.9968031048774719 - rollout_corr/training_ppl:1.1685254573822021 - rollout_corr/training_log_ppl:0.15316851437091827 - rollout_corr/kl:0.0011363307712599635 - rollout_corr/k3_kl:0.000975005968939513 - rollout_corr/rollout_ppl:1.1671303510665894 - rollout_corr/rollout_log_ppl:0.15201318264007568 - rollout_corr/log_ppl_diff:0.0011553284712135792 - rollout_corr/log_ppl_abs_diff:0.0032680965960025787 - rollout_corr/log_ppl_diff_max:0.046565696597099304 - rollout_corr/log_ppl_diff_min:-0.018364861607551575 - rollout_corr/ppl_ratio:1.0011656284332275 - rollout_corr/chi2_token:0.0016819238662719727 - rollout_corr/chi2_seq:0.22264599800109863 - actor/pg_clipfrac:0.002663928304779499 - actor/ppo_kl:-0.00015340460110143303 - actor/pg_clipfrac_lower:7.585411973802063e-06 - actor/pg_loss:0.0007893856381997466 - actor/loss:0.0007893862202763557 - actor/grad_norm:1.453125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.12739896774292 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:47 - training/epoch:0 - critic/score/mean:0.412109375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.412109375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008244550786912441 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008244550786912441 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.44140625 - response_length/max:128.0 - response_length/min:40.0 - response_length/clip_ratio:0.576171875 - response_length_non_aborted/mean:115.44140625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:40.0 - response_length_non_aborted/clip_ratio:0.576171875 - response/aborted_ratio:0.0 - prompt_length/mean:32.265625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.1015625 - timing_s/gen:8.187931244996435 - timing_s/old_log_prob:8.375964601000305 - timing_s/adv:0.3806320189978578 - timing_s/update_actor:26.681002823999734 - timing_s/update_weights:1.8687385489974986 - timing_s/step:45.67767499900219 - timing_per_token_ms/adv:0.0025165420556280764 - timing_per_token_ms/update_actor:0.17640099188109734 - timing_per_token_ms/gen:0.06926480598413388 - perf/total_num_tokens:151252 - perf/time_per_step:45.67767499900219 - perf/throughput:3311.289377213355 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 34%|███▍ | 48/140 [38:56<1:12:05, 47.01s/it] (TransferQueueController pid=2438) 2026-11-01 17:55:13,666 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 28671, Total req/min: 5487.24, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.23 minutes; (TransferQueueController pid=2438) Per-operation statistics: NOTIFY_DATA_UPDATE: req_count=7163, req/min=1370.90, avg_time=0.000079s, max_time=0.002164s, min_time=0.000030s; SET_CUSTOM_META: req_count=10706, req/min=2048.98, avg_time=0.000009s, max_time=0.000240s, min_time=0.000003s; KV_RETRIEVE_META: req_count=10747, req/min=2056.83, avg_time=0.000245s, max_time=0.063937s, min_time=0.000064s; CLEAR_META: req_count=14, req/min=2.68, avg_time=0.001668s, max_time=0.003645s, min_time=0.000783s; KV_LIST: req_count=35, req/min=6.70, avg_time=0.000238s, max_time=0.000665s, min_time=0.000045s; KV_RETRIEVE_KEYS: req_count=6, req/min=1.15, avg_time=0.000116s, max_time=0.000130s, min_time=0.000106s (TaskRunnerV1 pid=2351) Training Progress: 35%|███▌ | 49/140 [39:42<1:11:04, 46.87s/it] (SimpleStorageUnit pid=2599) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.18, avg_time=0.001360s, max_time=0.002711s, min_time=0.000417s; PUT_DATA: req_count=1046, req/min=208.08, avg_time=0.000606s, max_time=0.002337s, min_time=0.000303s; GET_DATA: req_count=33, req/min=6.56, avg_time=0.000664s, max_time=0.001209s, min_time=0.000424s (SimpleStorageUnit pid=2599) 2026-11-01 17:55:30,596 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1095, Total req/min: 217.83, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.03 minutes; (SimpleStorageUnit pid=2596) 2026-11-01 17:55:30,600 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_72b4f821: [Performance] Total success requests: 1075, Total req/min: 213.85, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2596) Time range: last 5.03 minutes; (TaskRunnerV1 pid=2351) Training Progress: 36%|███▌ | 50/140 [40:39<1:14:56, 49.96s/it] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.18, avg_time=0.001324s, max_time=0.004140s, min_time=0.000394s; PUT_DATA: req_count=1031, req/min=205.10, avg_time=0.000622s, max_time=0.002997s, min_time=0.000319s; GET_DATA: req_count=33, req/min=6.56, avg_time=0.000603s, max_time=0.001253s, min_time=0.000392s [repeated 7x across cluster] (SimpleStorageUnit pid=2583) 2026-11-01 17:55:30,598 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1080, Total req/min: 214.85, Total avg process time: 0.0006s; [repeated 6x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.03 minutes; [repeated 6x across cluster] (TaskRunnerV1 pid=2351) step:48 - global_seqlen/min:151684.0 - global_seqlen/max:151684.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151684.0 - global_seqlen/balanced_max:151684.0 - global_seqlen/mean:151684.0 - actor/entropy:0.14358636736869812 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6618123650550842 - training/rollout_probs_diff_mean:0.004862966947257519 - training/rollout_probs_diff_std:0.015997853130102158 - training/rollout_actor_probs_pearson_corr:0.9964620471000671 - rollout_corr/training_ppl:1.1650619506835938 - rollout_corr/training_log_ppl:0.14997442066669464 - rollout_corr/kl:0.0010565791744738817 - rollout_corr/k3_kl:0.0010639358079060912 - rollout_corr/rollout_ppl:1.1638767719268799 - rollout_corr/rollout_log_ppl:0.1489810049533844 - rollout_corr/log_ppl_diff:0.000993427005596459 - rollout_corr/log_ppl_abs_diff:0.003462795168161392 - rollout_corr/log_ppl_diff_max:0.02353622019290924 - rollout_corr/log_ppl_diff_min:-0.035437971353530884 - rollout_corr/ppl_ratio:1.0010051727294922 - rollout_corr/chi2_token:0.002932906150817871 - rollout_corr/chi2_seq:0.4404247999191284 - actor/pg_clipfrac:0.003855278033976598 - actor/ppo_kl:-2.260420221015616e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0004373387200757861 - actor/loss:-0.0004373411647975445 - actor/grad_norm:1.7578125 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.13334655761719 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:48 - training/epoch:0 - critic/score/mean:0.392578125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.392578125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005610356107354164 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005610356107354164 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:116.27734375 - response_length/max:128.0 - response_length/min:40.0 - response_length/clip_ratio:0.6123046875 - response_length_non_aborted/mean:116.27734375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:40.0 - response_length_non_aborted/clip_ratio:0.6123046875 - response/aborted_ratio:0.0 - prompt_length/mean:31.8515625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.21226148000278 - timing_s/old_log_prob:8.335136806002993 - timing_s/adv:0.1913978570009931 - timing_s/update_actor:27.022828731998743 - timing_s/update_weights:1.8665970439978992 - timing_s/step:45.79562091399566 - timing_per_token_ms/adv:0.0012618196843503144 - timing_per_token_ms/update_actor:0.17815213688984166 - timing_per_token_ms/gen:0.06897118856454111 - perf/total_num_tokens:151684 - perf/time_per_step:45.79562091399566 - perf/throughput:3312.1944188695047 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:49 - global_seqlen/min:154679.0 - global_seqlen/max:154679.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:154679.0 - global_seqlen/balanced_max:154679.0 - global_seqlen/mean:154679.0 - actor/entropy:0.1453452706336975 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8722193837165833 - training/rollout_probs_diff_mean:0.004608278162777424 - training/rollout_probs_diff_std:0.015212561935186386 - training/rollout_actor_probs_pearson_corr:0.9967471957206726 - rollout_corr/training_ppl:1.1646976470947266 - rollout_corr/training_log_ppl:0.1498529314994812 - rollout_corr/kl:0.0007458062609657645 - rollout_corr/k3_kl:0.0009102621115744114 - rollout_corr/rollout_ppl:1.1637479066848755 - rollout_corr/rollout_log_ppl:0.14907337725162506 - rollout_corr/log_ppl_diff:0.0007795357960276306 - rollout_corr/log_ppl_abs_diff:0.0032977499067783356 - rollout_corr/log_ppl_diff_max:0.01909855753183365 - rollout_corr/log_ppl_diff_min:-0.016758739948272705 - rollout_corr/ppl_ratio:1.0007894039154053 - rollout_corr/chi2_token:0.00215756893157959 - rollout_corr/chi2_seq:0.29500842094421387 - actor/pg_clipfrac:0.0032261895006134482 - actor/ppo_kl:2.500063878111785e-05 - actor/pg_clipfrac_lower:7.638252403315468e-06 - actor/pg_loss:-0.0007827937370166183 - actor/loss:-0.0007827915251255035 - actor/grad_norm:1.63671875 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.1176438331604 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:49 - training/epoch:0 - critic/score/mean:0.3310546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3310546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008255703374743462 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008255703374743462 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:117.9912109375 - response_length/max:128.0 - response_length/min:41.0 - response_length/clip_ratio:0.6630859375 - response_length_non_aborted/mean:117.9912109375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:41.0 - response_length_non_aborted/clip_ratio:0.6630859375 - response/aborted_ratio:0.0 - prompt_length/mean:33.0625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.192583469994133 - timing_s/old_log_prob:8.455509792001976 - timing_s/adv:0.1829295759962406 - timing_s/update_actor:27.290580286004115 - timing_s/update_weights:1.856970758002717 - timing_s/step:46.15157930400164 - timing_per_token_ms/adv:0.0011826400222153014 - timing_per_token_ms/update_actor:0.1764336483039334 - timing_per_token_ms/gen:0.06780648941008031 - perf/total_num_tokens:154679 - perf/time_per_step:46.15157930400164 - perf/throughput:3351.542944633063 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:50 - global_seqlen/min:154381.0 - global_seqlen/max:154381.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:154381.0 - global_seqlen/balanced_max:154381.0 - global_seqlen/mean:154381.0 - actor/entropy:0.14965496957302094 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6118559837341309 - training/rollout_probs_diff_mean:0.004723065998405218 - training/rollout_probs_diff_std:0.015207485295832157 - training/rollout_actor_probs_pearson_corr:0.9968138933181763 - rollout_corr/training_ppl:1.1717774868011475 - rollout_corr/training_log_ppl:0.1554202139377594 - rollout_corr/kl:0.0009423504816368222 - rollout_corr/k3_kl:0.0009596105082891881 - rollout_corr/rollout_ppl:1.1705926656723022 - rollout_corr/rollout_log_ppl:0.1544513702392578 - rollout_corr/log_ppl_diff:0.0009688293794170022 - rollout_corr/log_ppl_abs_diff:0.0032484768889844418 - rollout_corr/log_ppl_diff_max:0.03002724051475525 - rollout_corr/log_ppl_diff_min:-0.014066696166992188 - rollout_corr/ppl_ratio:1.0009788274765015 - rollout_corr/chi2_token:0.0018832683563232422 - rollout_corr/chi2_seq:0.23250293731689453 - actor/pg_clipfrac:0.0031795433921312606 - actor/ppo_kl:9.653201410249797e-05 - actor/pg_clipfrac_lower:7.783797021450786e-06 - actor/pg_loss:0.0012339178065303713 - actor/loss:0.001233917661011219 - actor/grad_norm:1.6357421875 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.05110454559326 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.423) - val-core/char_count/acc/mean@1:np.float64(0.423) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:50 - training/epoch:0 - critic/score/mean:0.3740234375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3740234375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007906596176326275 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007906596176326275 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:117.8955078125 - response_length/max:128.0 - response_length/min:30.0 - response_length/clip_ratio:0.640625 - response_length_non_aborted/mean:117.8955078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:30.0 - response_length_non_aborted/clip_ratio:0.640625 - response/aborted_ratio:0.0 - prompt_length/mean:32.8671875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.191699895003694 - timing_s/old_log_prob:8.439683281998441 - timing_s/adv:0.22743632200581487 - timing_s/update_actor:27.138077953000902 - timing_s/update_weights:1.8676303739994182 - timing_s/step:46.03657215999556 - timing_s/testing:11.017563882000104 - timing_per_token_ms/adv:0.0014732144629573255 - timing_per_token_ms/update_actor:0.1757863853259203 - timing_per_token_ms/gen:0.06785421325329215 - perf/total_num_tokens:154381 - perf/time_per_step:46.03657215999556 - perf/throughput:3353.44255135817 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 36%|███▋ | 51/140 [41:24<1:11:54, 48.48s/it] (TaskRunnerV1 pid=2351) Training Progress: 37%|███▋ | 52/140 [42:10<1:09:59, 47.72s/it] (TaskRunnerV1 pid=2351) Training Progress: 38%|███▊ | 53/140 [42:56<1:08:25, 47.19s/it] (TaskRunnerV1 pid=2351) step:51 - global_seqlen/min:147848.0 - global_seqlen/max:147848.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147848.0 - global_seqlen/balanced_max:147848.0 - global_seqlen/mean:147848.0 - actor/entropy:0.15169019997119904 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5360782742500305 - training/rollout_probs_diff_mean:0.0049833813682198524 - training/rollout_probs_diff_std:0.015358416363596916 - training/rollout_actor_probs_pearson_corr:0.9967756271362305 - rollout_corr/training_ppl:1.1762137413024902 - rollout_corr/training_log_ppl:0.15963461995124817 - rollout_corr/kl:0.0011429742444306612 - rollout_corr/k3_kl:0.000949490291532129 - rollout_corr/rollout_ppl:1.1746511459350586 - rollout_corr/rollout_log_ppl:0.15838032960891724 - rollout_corr/log_ppl_diff:0.0012542915064841509 - rollout_corr/log_ppl_abs_diff:0.003463746514171362 - rollout_corr/log_ppl_diff_max:0.034702956676483154 - rollout_corr/log_ppl_diff_min:-0.015063166618347168 - rollout_corr/ppl_ratio:1.0012662410736084 - rollout_corr/chi2_token:0.0014801025390625 - rollout_corr/chi2_seq:0.16233432292938232 - actor/pg_clipfrac:0.0029172318124134713 - actor/ppo_kl:2.7275760013859937e-05 - actor/pg_clipfrac_lower:7.632655372920757e-06 - actor/pg_loss:-9.951501851901412e-05 - actor/loss:-9.951554238796234e-05 - actor/grad_norm:1.9541015625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.129751682281494 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:51 - training/epoch:0 - critic/score/mean:0.4072265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4072265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008860615082085133 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008860615082085133 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:112.078125 - response_length/max:128.0 - response_length/min:34.0 - response_length/clip_ratio:0.5419921875 - response_length_non_aborted/mean:112.078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:34.0 - response_length_non_aborted/clip_ratio:0.5419921875 - response/aborted_ratio:0.0 - prompt_length/mean:32.3046875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.196243919999688 - timing_s/old_log_prob:8.242272244002379 - timing_s/adv:0.18442568400496384 - timing_s/update_actor:26.24422717699781 - timing_s/update_weights:1.8715627000055974 - timing_s/step:44.90217214099539 - timing_per_token_ms/adv:0.0012474006006504238 - timing_per_token_ms/update_actor:0.17750816498699887 - timing_per_token_ms/gen:0.07141575979366799 - perf/total_num_tokens:147848 - perf/time_per_step:44.90217214099539 - perf/throughput:3292.669217332044 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:52 - global_seqlen/min:152790.0 - global_seqlen/max:152790.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:152790.0 - global_seqlen/balanced_max:152790.0 - global_seqlen/mean:152790.0 - actor/entropy:0.14310140907764435 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8045268058776855 - training/rollout_probs_diff_mean:0.004617778584361076 - training/rollout_probs_diff_std:0.016227472573518753 - training/rollout_actor_probs_pearson_corr:0.9963238835334778 - rollout_corr/training_ppl:1.162857174873352 - rollout_corr/training_log_ppl:0.14847460389137268 - rollout_corr/kl:0.0009509353549219668 - rollout_corr/k3_kl:0.0010377684375271201 - rollout_corr/rollout_ppl:1.161673665046692 - rollout_corr/rollout_log_ppl:0.14749324321746826 - rollout_corr/log_ppl_diff:0.0009813618380576372 - rollout_corr/log_ppl_abs_diff:0.0033755325712263584 - rollout_corr/log_ppl_diff_max:0.0250169038772583 - rollout_corr/log_ppl_diff_min:-0.020777404308319092 - rollout_corr/ppl_ratio:1.0009920597076416 - rollout_corr/chi2_token:0.0027936697006225586 - rollout_corr/chi2_seq:0.31815481185913086 - actor/pg_clipfrac:0.0028433674985232455 - actor/ppo_kl:0.00016283161289720738 - actor/pg_clipfrac_lower:2.0865801161562558e-05 - actor/pg_loss:0.00018468961934559047 - actor/loss:0.00018469011411070824 - actor/grad_norm:1.732421875 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.14779996871948 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:52 - training/epoch:0 - critic/score/mean:0.365234375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.365234375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.01208979357033968 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.01208979357033968 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:116.380859375 - response_length/max:128.0 - response_length/min:33.0 - response_length/clip_ratio:0.6201171875 - response_length_non_aborted/mean:116.380859375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:33.0 - response_length_non_aborted/clip_ratio:0.6201171875 - response/aborted_ratio:0.0 - prompt_length/mean:32.828125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.195201425005507 - timing_s/old_log_prob:8.328858302003937 - timing_s/adv:0.18245332600054098 - timing_s/update_actor:26.889150224997138 - timing_s/update_weights:1.8705697559998953 - timing_s/step:45.64161216599314 - timing_per_token_ms/adv:0.001194144420449905 - timing_per_token_ms/update_actor:0.17598763155309338 - timing_per_token_ms/gen:0.06876668925273556 - perf/total_num_tokens:152790 - perf/time_per_step:45.64161216599314 - perf/throughput:3347.6030479449514 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:53 - global_seqlen/min:152653.0 - global_seqlen/max:152653.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:152653.0 - global_seqlen/balanced_max:152653.0 - global_seqlen/mean:152653.0 - actor/entropy:0.14128378033638 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8638778328895569 - training/rollout_probs_diff_mean:0.004578258842229843 - training/rollout_probs_diff_std:0.015667108818888664 - training/rollout_actor_probs_pearson_corr:0.9964109063148499 - rollout_corr/training_ppl:1.1590096950531006 - rollout_corr/training_log_ppl:0.145355686545372 - rollout_corr/kl:0.000820876331999898 - rollout_corr/k3_kl:0.0010220831027254462 - rollout_corr/rollout_ppl:1.1580133438110352 - rollout_corr/rollout_log_ppl:0.14451685547828674 - rollout_corr/log_ppl_diff:0.0008388287387788296 - rollout_corr/log_ppl_abs_diff:0.0033480955753475428 - rollout_corr/log_ppl_diff_max:0.037216395139694214 - rollout_corr/log_ppl_diff_min:-0.016693130135536194 - rollout_corr/ppl_ratio:1.000849723815918 - rollout_corr/chi2_token:0.0025725364685058594 - rollout_corr/chi2_seq:0.3665030002593994 - actor/pg_clipfrac:0.0029407300292708287 - actor/ppo_kl:0.0001711635046982944 - actor/pg_clipfrac_lower:7.571627672101992e-06 - actor/pg_loss:-0.0014858614886179566 - actor/loss:-0.0014858613722026348 - actor/grad_norm:1.6201171875 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.0695686340332 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:53 - training/epoch:0 - critic/score/mean:0.3701171875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3701171875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008421557024121284 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008421557024121284 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:116.2783203125 - response_length/max:128.0 - response_length/min:42.0 - response_length/clip_ratio:0.6064453125 - response_length_non_aborted/mean:116.2783203125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:42.0 - response_length_non_aborted/clip_ratio:0.6064453125 - response/aborted_ratio:0.0 - prompt_length/mean:32.796875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.210391546999745 - timing_s/old_log_prob:8.346153748003417 - timing_s/adv:0.18872612099949038 - timing_s/update_actor:26.961455680000654 - timing_s/update_weights:1.89937938300136 - timing_s/step:45.772974995998084 - timing_per_token_ms/adv:0.0012363079729811428 - timing_per_token_ms/update_actor:0.17661923237670177 - timing_per_token_ms/gen:0.06895490469391483 - perf/total_num_tokens:152653 - perf/time_per_step:45.772974995998084 - perf/throughput:3335.002804894076 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 39%|███▊ | 54/140 [43:42<1:07:07, 46.83s/it] (TaskRunnerV1 pid=2351) Training Progress: 39%|███▉ | 55/140 [44:36<1:09:26, 49.02s/it] (TransferQueueController pid=2438) 2026-11-01 18:00:13,697 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 36707, Total req/min: 7340.64, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.00 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=14136, req/min=2826.91, avg_time=0.000236s, max_time=0.070579s, min_time=0.000063s; CLEAR_META: req_count=17, req/min=3.40, avg_time=0.002043s, max_time=0.003818s, min_time=0.000943s; SET_CUSTOM_META: req_count=14090, req/min=2817.71, avg_time=0.000009s, max_time=0.000733s, min_time=0.000003s; KV_LIST: req_count=44, req/min=8.80, avg_time=0.000271s, max_time=0.001044s, min_time=0.000049s; NOTIFY_DATA_UPDATE: req_count=8414, req/min=1682.63, avg_time=0.000078s, max_time=0.001182s, min_time=0.000030s; KV_RETRIEVE_KEYS: req_count=6, req/min=1.20, avg_time=0.000149s, max_time=0.000241s, min_time=0.000112s (SimpleStorageUnit pid=2599) Per-operation statistics: GET_DATA: req_count=38, req/min=7.05, avg_time=0.000587s, max_time=0.001165s, min_time=0.000375s; PUT_DATA: req_count=1068, req/min=198.19, avg_time=0.000580s, max_time=0.001461s, min_time=0.000317s; CLEAR_DATA: req_count=16, req/min=2.97, avg_time=0.001293s, max_time=0.002858s, min_time=0.000393s (SimpleStorageUnit pid=2599) 2026-11-01 18:00:53,918 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1122, Total req/min: 208.21, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.39 minutes; (SimpleStorageUnit pid=2596) 2026-11-01 18:00:53,918 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_72b4f821: [Performance] Total success requests: 1095, Total req/min: 203.21, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2596) Time range: last 5.39 minutes; (TaskRunnerV1 pid=2351) Training Progress: 40%|████ | 56/140 [45:22<1:07:15, 48.04s/it] (TaskRunnerV1 pid=2351) step:54 - global_seqlen/min:152993.0 - global_seqlen/max:152993.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:152993.0 - global_seqlen/balanced_max:152993.0 - global_seqlen/mean:152993.0 - actor/entropy:0.14407847821712494 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.4668270945549011 - training/rollout_probs_diff_mean:0.0046288021840155125 - training/rollout_probs_diff_std:0.014656390063464642 - training/rollout_actor_probs_pearson_corr:0.9968888759613037 - rollout_corr/training_ppl:1.1634148359298706 - rollout_corr/training_log_ppl:0.14883394539356232 - rollout_corr/kl:0.0010120593942701817 - rollout_corr/k3_kl:0.0008771036518737674 - rollout_corr/rollout_ppl:1.1621173620224 - rollout_corr/rollout_log_ppl:0.14777672290802002 - rollout_corr/log_ppl_diff:0.0010572181781753898 - rollout_corr/log_ppl_abs_diff:0.0032403054647147655 - rollout_corr/log_ppl_diff_max:0.029921263456344604 - rollout_corr/log_ppl_diff_min:-0.01625622808933258 - rollout_corr/ppl_ratio:1.00106680393219 - rollout_corr/chi2_token:0.0014814138412475586 - rollout_corr/chi2_seq:0.17850351333618164 - actor/pg_clipfrac:0.003135079057149899 - actor/ppo_kl:-0.00014612183076678775 - actor/pg_clipfrac_lower:3.134798528966106e-05 - actor/pg_loss:0.001539949094876647 - actor/loss:0.001539949793368578 - actor/grad_norm:1.7275390625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.18367385864258 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:54 - training/epoch:0 - critic/score/mean:0.3876953125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3876953125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006342541426420212 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006342541426420212 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.9072265625 - response_length/max:128.0 - response_length/min:45.0 - response_length/clip_ratio:0.61328125 - response_length_non_aborted/mean:115.9072265625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:45.0 - response_length_non_aborted/clip_ratio:0.61328125 - response/aborted_ratio:0.0 - prompt_length/mean:33.5 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:8.193344378996699 - timing_s/old_log_prob:8.382303970996873 - timing_s/adv:0.18079882700112648 - timing_s/update_actor:27.018874922003306 - timing_s/update_weights:1.9090756610021344 - timing_s/step:45.85653500100307 - timing_per_token_ms/adv:0.0011817457465447862 - timing_per_token_ms/update_actor:0.17660203357018495 - timing_per_token_ms/gen:0.06903204491567626 - perf/total_num_tokens:152993 - perf/time_per_step:45.85653500100307 - perf/throughput:3336.340174778871 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:55 - global_seqlen/min:147909.0 - global_seqlen/max:147909.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147909.0 - global_seqlen/balanced_max:147909.0 - global_seqlen/mean:147909.0 - actor/entropy:0.14574675261974335 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5830116271972656 - training/rollout_probs_diff_mean:0.004699762910604477 - training/rollout_probs_diff_std:0.015115446411073208 - training/rollout_actor_probs_pearson_corr:0.9968217611312866 - rollout_corr/training_ppl:1.1661770343780518 - rollout_corr/training_log_ppl:0.15149091184139252 - rollout_corr/kl:0.0009385589510202408 - rollout_corr/k3_kl:0.000885047425981611 - rollout_corr/rollout_ppl:1.1650854349136353 - rollout_corr/rollout_log_ppl:0.15059222280979156 - rollout_corr/log_ppl_diff:0.0008986982866190374 - rollout_corr/log_ppl_abs_diff:0.0033526532351970673 - rollout_corr/log_ppl_diff_max:0.021073490381240845 - rollout_corr/log_ppl_diff_min:-0.015856757760047913 - rollout_corr/ppl_ratio:1.000908613204956 - rollout_corr/chi2_token:0.0016777515411376953 - rollout_corr/chi2_seq:0.20214879512786865 - actor/pg_clipfrac:0.002866726616048254 - actor/ppo_kl:-5.2298964874353124e-06 - actor/pg_clipfrac_lower:1.6871504461353954e-05 - actor/pg_loss:0.0008322609646711498 - actor/loss:0.0008322601206600666 - actor/grad_norm:1.541015625 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.15022850036621 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.436) - val-core/char_count/acc/mean@1:np.float64(0.436) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:55 - training/epoch:0 - critic/score/mean:0.4111328125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4111328125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005242159590125084 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005242159590125084 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:112.1533203125 - response_length/max:128.0 - response_length/min:35.0 - response_length/clip_ratio:0.5419921875 - response_length_non_aborted/mean:112.1533203125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:35.0 - response_length_non_aborted/clip_ratio:0.5419921875 - response/aborted_ratio:0.0 - prompt_length/mean:32.2890625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.210709860999486 - timing_s/old_log_prob:8.482247016996553 - timing_s/adv:0.1817012129977229 - timing_s/update_actor:26.15523876700172 - timing_s/update_weights:1.8632760769978631 - timing_s/step:45.05947109199769 - timing_s/testing:8.718625557994528 - timing_per_token_ms/adv:0.001228466239361519 - timing_per_token_ms/update_actor:0.17683331485576753 - timing_per_token_ms/gen:0.0714938383125037 - perf/total_num_tokens:147909 - perf/time_per_step:45.05947109199769 - perf/throughput:3282.528543178302 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:56 - global_seqlen/min:151684.0 - global_seqlen/max:151684.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151684.0 - global_seqlen/balanced_max:151684.0 - global_seqlen/mean:151684.0 - actor/entropy:0.14081056416034698 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6724858283996582 - training/rollout_probs_diff_mean:0.004516386892646551 - training/rollout_probs_diff_std:0.014854595996439457 - training/rollout_actor_probs_pearson_corr:0.9967657923698425 - rollout_corr/training_ppl:1.1592999696731567 - rollout_corr/training_log_ppl:0.14561840891838074 - rollout_corr/kl:0.0009603013750165701 - rollout_corr/k3_kl:0.0009641785873100162 - rollout_corr/rollout_ppl:1.15810227394104 - rollout_corr/rollout_log_ppl:0.14463241398334503 - rollout_corr/log_ppl_diff:0.0009859958663582802 - rollout_corr/log_ppl_abs_diff:0.003275643801316619 - rollout_corr/log_ppl_diff_max:0.03917188197374344 - rollout_corr/log_ppl_diff_min:-0.023740962147712708 - rollout_corr/ppl_ratio:1.000996470451355 - rollout_corr/chi2_token:0.0025529861450195312 - rollout_corr/chi2_seq:0.3551309108734131 - actor/pg_clipfrac:0.0031639545098490394 - actor/ppo_kl:-2.0696144929388524e-05 - actor/pg_clipfrac_lower:1.5326384527725168e-05 - actor/pg_loss:-0.001258898206287995 - actor/loss:-0.0012588986428454518 - actor/grad_norm:1.693359375 - actor/perf/max_memory_allocated_gb:21.653789520263672 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.067355155944824 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:56 - training/epoch:0 - critic/score/mean:0.4375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004307485651224852 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004307485651224852 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.67578125 - response_length/max:128.0 - response_length/min:44.0 - response_length/clip_ratio:0.5869140625 - response_length_non_aborted/mean:115.67578125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:44.0 - response_length_non_aborted/clip_ratio:0.5869140625 - response/aborted_ratio:0.0 - prompt_length/mean:32.453125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.191388320999977 - timing_s/old_log_prob:8.3833320440026 - timing_s/adv:0.1807979119985248 - timing_s/update_actor:26.817551641004684 - timing_s/update_weights:1.884232753996912 - timing_s/step:45.64669889700599 - timing_per_token_ms/adv:0.0011919379235682391 - timing_per_token_ms/update_actor:0.1767988162298244 - timing_per_token_ms/gen:0.06915365144531098 - perf/total_num_tokens:151684 - perf/time_per_step:45.64669889700599 - perf/throughput:3323.0004286235276 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 41%|████ | 57/140 [46:08<1:05:30, 47.35s/it] (SimpleStorageUnit pid=2583) Per-operation statistics: GET_DATA: req_count=38, req/min=7.05, avg_time=0.000598s, max_time=0.000939s, min_time=0.000396s; PUT_DATA: req_count=1052, req/min=195.23, avg_time=0.000625s, max_time=0.002444s, min_time=0.000312s; CLEAR_DATA: req_count=16, req/min=2.97, avg_time=0.001615s, max_time=0.004504s, min_time=0.000416s [repeated 7x across cluster] (SimpleStorageUnit pid=2583) 2026-11-01 18:00:53,917 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1106, Total req/min: 205.25, Total avg process time: 0.0006s; [repeated 6x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.39 minutes; [repeated 6x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 41%|████▏ | 58/140 [46:53<1:03:52, 46.74s/it] (TaskRunnerV1 pid=2351) Training Progress: 42%|████▏ | 59/140 [47:39<1:02:53, 46.59s/it] (TaskRunnerV1 pid=2351) step:57 - global_seqlen/min:150182.0 - global_seqlen/max:150182.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:150182.0 - global_seqlen/balanced_max:150182.0 - global_seqlen/mean:150182.0 - actor/entropy:0.14323945343494415 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8378893136978149 - training/rollout_probs_diff_mean:0.004588656593114138 - training/rollout_probs_diff_std:0.014999950304627419 - training/rollout_actor_probs_pearson_corr:0.9967687726020813 - rollout_corr/training_ppl:1.1649821996688843 - rollout_corr/training_log_ppl:0.1500786989927292 - rollout_corr/kl:0.0009733387269079685 - rollout_corr/k3_kl:0.0009625632083043456 - rollout_corr/rollout_ppl:1.16383957862854 - rollout_corr/rollout_log_ppl:0.14911814033985138 - rollout_corr/log_ppl_diff:0.000960546953137964 - rollout_corr/log_ppl_abs_diff:0.003386218100786209 - rollout_corr/log_ppl_diff_max:0.041269272565841675 - rollout_corr/log_ppl_diff_min:-0.01285696029663086 - rollout_corr/ppl_ratio:1.000971794128418 - rollout_corr/chi2_token:0.0016982555389404297 - rollout_corr/chi2_seq:0.23193883895874023 - actor/pg_clipfrac:0.00383018450520467 - actor/ppo_kl:0.00013319884639410398 - actor/pg_clipfrac_lower:1.514050381956622e-05 - actor/pg_loss:0.0030561225721612573 - actor/loss:0.0030561243183910847 - actor/grad_norm:1.7802734375 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.058605670928955 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:57 - training/epoch:0 - critic/score/mean:0.41796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.41796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004147508181631565 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004147508181631565 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:114.013671875 - response_length/max:128.0 - response_length/min:32.0 - response_length/clip_ratio:0.5634765625 - response_length_non_aborted/mean:114.013671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:32.0 - response_length_non_aborted/clip_ratio:0.5634765625 - response/aborted_ratio:0.0 - prompt_length/mean:32.6484375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:8.211771076996229 - timing_s/old_log_prob:8.470937821999541 - timing_s/adv:0.19336333299725084 - timing_s/update_actor:26.642621966995648 - timing_s/update_weights:1.886546358000487 - timing_s/step:45.571413698999095 - timing_per_token_ms/adv:0.0012875266876007168 - timing_per_token_ms/update_actor:0.17740223173879457 - timing_per_token_ms/gen:0.07033636896784778 - perf/total_num_tokens:150182 - perf/time_per_step:45.571413698999095 - perf/throughput:3295.5308560747703 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:58 - global_seqlen/min:147769.0 - global_seqlen/max:147769.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147769.0 - global_seqlen/balanced_max:147769.0 - global_seqlen/mean:147769.0 - actor/entropy:0.14865495264530182 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9465715289115906 - training/rollout_probs_diff_mean:0.004947981797158718 - training/rollout_probs_diff_std:0.01697828806936741 - training/rollout_actor_probs_pearson_corr:0.9960388541221619 - rollout_corr/training_ppl:1.1687077283859253 - rollout_corr/training_log_ppl:0.1532580852508545 - rollout_corr/kl:0.0011712462874129415 - rollout_corr/k3_kl:0.00112428842112422 - rollout_corr/rollout_ppl:1.167305827140808 - rollout_corr/rollout_log_ppl:0.15209370851516724 - rollout_corr/log_ppl_diff:0.0011643890757113695 - rollout_corr/log_ppl_abs_diff:0.0035967689473181963 - rollout_corr/log_ppl_diff_max:0.055394500494003296 - rollout_corr/log_ppl_diff_min:-0.02671535313129425 - rollout_corr/ppl_ratio:1.0011792182922363 - rollout_corr/chi2_token:0.0019878149032592773 - rollout_corr/chi2_seq:0.22163057327270508 - actor/pg_clipfrac:0.0030336669296957552 - actor/ppo_kl:9.325206300066688e-05 - actor/pg_clipfrac_lower:1.557686967620005e-05 - actor/pg_loss:0.0003250991285312921 - actor/loss:0.0003250997979193926 - actor/grad_norm:1.935546875 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.05313158035278 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:58 - training/epoch:0 - critic/score/mean:0.47265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.47265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.003267040941864252 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.003267040941864252 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:112.7197265625 - response_length/max:128.0 - response_length/min:39.0 - response_length/clip_ratio:0.5166015625 - response_length_non_aborted/mean:112.7197265625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:39.0 - response_length_non_aborted/clip_ratio:0.5166015625 - response/aborted_ratio:0.0 - prompt_length/mean:31.5859375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.215011729997059 - timing_s/old_log_prob:8.502678128003026 - timing_s/adv:0.18174052699760068 - timing_s/update_actor:26.249797294003656 - timing_s/update_weights:1.8713112059995183 - timing_s/step:45.18871595100063 - timing_per_token_ms/adv:0.0012298961690043288 - timing_per_token_ms/update_actor:0.17764075884660285 - timing_per_token_ms/gen:0.07117185817627948 - perf/total_num_tokens:147769 - perf/time_per_step:45.18871595100063 - perf/throughput:3270.0420202297846 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:59 - global_seqlen/min:152785.0 - global_seqlen/max:152785.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:152785.0 - global_seqlen/balanced_max:152785.0 - global_seqlen/mean:152785.0 - actor/entropy:0.14079828560352325 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8673208951950073 - training/rollout_probs_diff_mean:0.00480868062004447 - training/rollout_probs_diff_std:0.016604645177721977 - training/rollout_actor_probs_pearson_corr:0.9960305690765381 - rollout_corr/training_ppl:1.161240816116333 - rollout_corr/training_log_ppl:0.1469619870185852 - rollout_corr/kl:0.0012051078956574202 - rollout_corr/k3_kl:0.0010827678488567472 - rollout_corr/rollout_ppl:1.159724235534668 - rollout_corr/rollout_log_ppl:0.1457153558731079 - rollout_corr/log_ppl_diff:0.0012466155458241701 - rollout_corr/log_ppl_abs_diff:0.0035458686761558056 - rollout_corr/log_ppl_diff_max:0.0447855144739151 - rollout_corr/log_ppl_diff_min:-0.02218806743621826 - rollout_corr/ppl_ratio:1.001259446144104 - rollout_corr/chi2_token:0.0018510818481445312 - rollout_corr/chi2_seq:0.21956658363342285 - actor/pg_clipfrac:0.0033885643691367777 - actor/ppo_kl:-0.00019512175770349435 - actor/pg_clipfrac_lower:4.155459676743097e-05 - actor/pg_loss:0.003102181770373136 - actor/loss:0.003102181712165475 - actor/grad_norm:1.8271484375 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.115798473358154 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:59 - training/epoch:0 - critic/score/mean:0.3056640625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3056640625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00025934871518984437 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.00025934871518984437 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.9619140625 - response_length/max:128.0 - response_length/min:38.0 - response_length/clip_ratio:0.6044921875 - response_length_non_aborted/mean:115.9619140625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:38.0 - response_length_non_aborted/clip_ratio:0.6044921875 - response/aborted_ratio:0.0 - prompt_length/mean:33.2421875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.20844087900332 - timing_s/old_log_prob:8.337363965001714 - timing_s/adv:0.4574210179998772 - timing_s/update_actor:27.04362749300344 - timing_s/update_weights:1.8823219740006607 - timing_s/step:46.09418659799849 - timing_per_token_ms/adv:0.0029938869522523628 - timing_per_token_ms/update_actor:0.1770044670157636 - timing_per_token_ms/gen:0.0691266232599547 - perf/total_num_tokens:152785 - perf/time_per_step:46.09418659799849 - perf/throughput:3314.626231122913 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 43%|████▎ | 60/140 [48:34<1:05:20, 49.01s/it] (TaskRunnerV1 pid=2351) Training Progress: 44%|████▎ | 61/140 [49:20<1:03:15, 48.05s/it] (TransferQueueController pid=2438) 2026-11-01 18:05:36,332 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 31250, Total req/min: 5811.52, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.38 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=11649, req/min=2166.35, avg_time=0.000259s, max_time=0.070708s, min_time=0.000064s; NOTIFY_DATA_UPDATE: req_count=7937, req/min=1476.03, avg_time=0.000080s, max_time=0.000860s, min_time=0.000031s; SET_CUSTOM_META: req_count=11605, req/min=2158.17, avg_time=0.000009s, max_time=0.000266s, min_time=0.000003s; KV_LIST: req_count=37, req/min=6.88, avg_time=0.000243s, max_time=0.000854s, min_time=0.000041s; CLEAR_META: req_count=15, req/min=2.79, avg_time=0.001671s, max_time=0.003861s, min_time=0.000834s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.30, avg_time=0.000133s, max_time=0.000216s, min_time=0.000108s (TaskRunnerV1 pid=2351) Training Progress: 44%|████▍ | 62/140 [50:05<1:01:08, 47.04s/it] (TaskRunnerV1 pid=2351) step:60 - global_seqlen/min:151934.0 - global_seqlen/max:151934.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151934.0 - global_seqlen/balanced_max:151934.0 - global_seqlen/mean:151934.0 - actor/entropy:0.13872213661670685 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8339666724205017 - training/rollout_probs_diff_mean:0.004575548693537712 - training/rollout_probs_diff_std:0.01524176076054573 - training/rollout_actor_probs_pearson_corr:0.9966492652893066 - rollout_corr/training_ppl:1.157358169555664 - rollout_corr/training_log_ppl:0.1438683122396469 - rollout_corr/kl:0.0008620259468443692 - rollout_corr/k3_kl:0.0009008951019495726 - rollout_corr/rollout_ppl:1.156363606452942 - rollout_corr/rollout_log_ppl:0.14302833378314972 - rollout_corr/log_ppl_diff:0.0008399735670536757 - rollout_corr/log_ppl_abs_diff:0.0032103187404572964 - rollout_corr/log_ppl_diff_max:0.026647433638572693 - rollout_corr/log_ppl_diff_min:-0.01695774495601654 - rollout_corr/ppl_ratio:1.0008493661880493 - rollout_corr/chi2_token:0.0018557310104370117 - rollout_corr/chi2_seq:0.2858846187591553 - actor/pg_clipfrac:0.0029938184646501513 - actor/ppo_kl:-1.963778390745574e-05 - actor/pg_clipfrac_lower:7.753380486974493e-06 - actor/pg_loss:0.0029403263524727663 - actor/loss:0.0029403255321085453 - actor/grad_norm:1.69140625 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.059977531433105 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.437) - val-core/char_count/acc/mean@1:np.float64(0.437) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:60 - training/epoch:0 - critic/score/mean:0.3984375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3984375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004660234786570072 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004660234786570072 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.552734375 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.5908203125 - response_length_non_aborted/mean:115.552734375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.5908203125 - response/aborted_ratio:0.0 - prompt_length/mean:32.8203125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.211747376000858 - timing_s/old_log_prob:8.356262890003563 - timing_s/adv:0.1814355489987065 - timing_s/update_actor:27.042893960002402 - timing_s/update_weights:1.8845102740015136 - timing_s/step:45.84626144199865 - timing_s/testing:8.6900615630002 - timing_per_token_ms/adv:0.0011941734503054385 - timing_per_token_ms/update_actor:0.17799106164520384 - timing_per_token_ms/gen:0.0693993490526246 - perf/total_num_tokens:151934 - perf/time_per_step:45.84626144199865 - perf/throughput:3313.988866730515 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:61 - global_seqlen/min:151648.0 - global_seqlen/max:151648.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151648.0 - global_seqlen/balanced_max:151648.0 - global_seqlen/mean:151648.0 - actor/entropy:0.1401037722826004 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.635709285736084 - training/rollout_probs_diff_mean:0.004595104139298201 - training/rollout_probs_diff_std:0.01539552677422762 - training/rollout_actor_probs_pearson_corr:0.9966168999671936 - rollout_corr/training_ppl:1.1581857204437256 - rollout_corr/training_log_ppl:0.1443176418542862 - rollout_corr/kl:0.0010037064785137773 - rollout_corr/k3_kl:0.0009506286005489528 - rollout_corr/rollout_ppl:1.156943440437317 - rollout_corr/rollout_log_ppl:0.14328354597091675 - rollout_corr/log_ppl_diff:0.001034098444506526 - rollout_corr/log_ppl_abs_diff:0.0033248420804739 - rollout_corr/log_ppl_diff_max:0.04500773549079895 - rollout_corr/log_ppl_diff_min:-0.015504390001296997 - rollout_corr/ppl_ratio:1.0010449886322021 - rollout_corr/chi2_token:0.0016980171203613281 - rollout_corr/chi2_seq:0.2037109136581421 - actor/pg_clipfrac:0.0029484493667647866 - actor/ppo_kl:-8.298832017317181e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0001031620122375898 - actor/loss:-0.0001031619030982256 - actor/grad_norm:1.66796875 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.06434965133667 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:61 - training/epoch:0 - critic/score/mean:0.42578125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.42578125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006028711795806885 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006028711795806885 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.9375 - response_length/max:128.0 - response_length/min:35.0 - response_length/clip_ratio:0.5869140625 - response_length_non_aborted/mean:115.9375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:35.0 - response_length_non_aborted/clip_ratio:0.5869140625 - response/aborted_ratio:0.0 - prompt_length/mean:32.15625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.202529362002679 - timing_s/old_log_prob:8.554211842005316 - timing_s/adv:0.18266636900079902 - timing_s/update_actor:26.69984296400071 - timing_s/update_weights:1.8617139089983539 - timing_s/step:45.669158459000755 - timing_per_token_ms/adv:0.0012045418930734268 - timing_per_token_ms/update_actor:0.17606459012977888 - timing_per_token_ms/gen:0.06909138613546731 - perf/total_num_tokens:151648 - perf/time_per_step:45.669158459000755 - perf/throughput:3320.5779374310387 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:62 - global_seqlen/min:146803.0 - global_seqlen/max:146803.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146803.0 - global_seqlen/balanced_max:146803.0 - global_seqlen/mean:146803.0 - actor/entropy:0.1470358967781067 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9625489115715027 - training/rollout_probs_diff_mean:0.004811161197721958 - training/rollout_probs_diff_std:0.016233032569289207 - training/rollout_actor_probs_pearson_corr:0.9964147806167603 - rollout_corr/training_ppl:1.1700729131698608 - rollout_corr/training_log_ppl:0.15414626896381378 - rollout_corr/kl:0.0010719456477090716 - rollout_corr/k3_kl:0.0011235374258831143 - rollout_corr/rollout_ppl:1.16866135597229 - rollout_corr/rollout_log_ppl:0.15300077199935913 - rollout_corr/log_ppl_diff:0.001145506976172328 - rollout_corr/log_ppl_abs_diff:0.0038208921905606985 - rollout_corr/log_ppl_diff_max:0.10325959324836731 - rollout_corr/log_ppl_diff_min:-0.017888814210891724 - rollout_corr/ppl_ratio:1.001164436340332 - rollout_corr/chi2_token:0.002211928367614746 - rollout_corr/chi2_seq:0.3698611259460449 - actor/pg_clipfrac:0.003799979759302611 - actor/ppo_kl:-4.559093872558151e-05 - actor/pg_clipfrac_lower:1.8597069356474094e-05 - actor/pg_loss:7.520421058870852e-05 - actor/loss:7.520546205341816e-05 - actor/grad_norm:1.8603515625 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.109561920166016 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:62 - training/epoch:0 - critic/score/mean:0.455078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.455078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008645066060125828 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008645066060125828 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:111.3310546875 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.5107421875 - response_length_non_aborted/mean:111.3310546875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.5107421875 - response/aborted_ratio:0.0 - prompt_length/mean:32.03125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.188895147999574 - timing_s/old_log_prob:8.284890830000222 - timing_s/adv:0.20205438999983016 - timing_s/update_actor:25.82161189600447 - timing_s/update_weights:1.8851749769964954 - timing_s/step:44.55192975099635 - timing_per_token_ms/adv:0.0013763641751178802 - timing_per_token_ms/update_actor:0.17589294425866275 - timing_per_token_ms/gen:0.07183052330201463 - perf/total_num_tokens:146803 - perf/time_per_step:44.55192975099635 - perf/throughput:3295.098569702627 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(SimpleStorageUnit pid=2599) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=2.93, avg_time=0.001430s, max_time=0.003544s, min_time=0.000414s; PUT_DATA: req_count=1054, req/min=192.86, avg_time=0.000616s, max_time=0.001963s, min_time=0.000310s; GET_DATA: req_count=37, req/min=6.77, avg_time=0.000619s, max_time=0.001178s, min_time=0.000401s (SimpleStorageUnit pid=2599) 2026-11-01 18:06:21,823 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1107, Total req/min: 202.56, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.47 minutes; (SimpleStorageUnit pid=2596) 2026-11-01 18:06:21,824 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_72b4f821: [Performance] Total success requests: 1103, Total req/min: 201.83, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2596) Time range: last 5.47 minutes; (TaskRunnerV1 pid=2351) Training Progress: 45%|████▌ | 63/140 [50:50<59:46, 46.57s/it] (TaskRunnerV1 pid=2351) Training Progress: 46%|████▌ | 64/140 [51:36<58:45, 46.39s/it] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=2.93, avg_time=0.001362s, max_time=0.004273s, min_time=0.000381s; PUT_DATA: req_count=1040, req/min=190.30, avg_time=0.000579s, max_time=0.001612s, min_time=0.000304s; GET_DATA: req_count=37, req/min=6.77, avg_time=0.000625s, max_time=0.001242s, min_time=0.000393s [repeated 7x across cluster] (SimpleStorageUnit pid=2583) 2026-11-01 18:06:21,823 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1093, Total req/min: 200.00, Total avg process time: 0.0006s; [repeated 6x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.47 minutes; [repeated 6x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 46%|████▋ | 65/140 [52:30<1:00:53, 48.71s/it] (TaskRunnerV1 pid=2351) step:63 - global_seqlen/min:148603.0 - global_seqlen/max:148603.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:148603.0 - global_seqlen/balanced_max:148603.0 - global_seqlen/mean:148603.0 - actor/entropy:0.13800129294395447 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9928532242774963 - training/rollout_probs_diff_mean:0.004471357446163893 - training/rollout_probs_diff_std:0.015040921047329903 - training/rollout_actor_probs_pearson_corr:0.9967454075813293 - rollout_corr/training_ppl:1.1578339338302612 - rollout_corr/training_log_ppl:0.14443451166152954 - rollout_corr/kl:0.0009517431608401239 - rollout_corr/k3_kl:0.0009444167953915894 - rollout_corr/rollout_ppl:1.1567057371139526 - rollout_corr/rollout_log_ppl:0.14348909258842468 - rollout_corr/log_ppl_diff:0.0009454275132156909 - rollout_corr/log_ppl_abs_diff:0.0033933601807802916 - rollout_corr/log_ppl_diff_max:0.036534637212753296 - rollout_corr/log_ppl_diff_min:-0.02573978900909424 - rollout_corr/ppl_ratio:1.0009560585021973 - rollout_corr/chi2_token:0.0018714666366577148 - rollout_corr/chi2_seq:0.3123708963394165 - actor/pg_clipfrac:0.0027779804219486928 - actor/ppo_kl:6.678290075493958e-05 - actor/pg_clipfrac_lower:8.93367662987051e-06 - actor/pg_loss:0.0009013523813337088 - actor/loss:0.0009013500530272722 - actor/grad_norm:1.6748046875 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.156362533569336 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:63 - training/epoch:0 - critic/score/mean:0.4384765625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4384765625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008072390221059322 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008072390221059322 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:112.6591796875 - response_length/max:128.0 - response_length/min:36.0 - response_length/clip_ratio:0.5224609375 - response_length_non_aborted/mean:112.6591796875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:36.0 - response_length_non_aborted/clip_ratio:0.5224609375 - response/aborted_ratio:0.0 - prompt_length/mean:32.4609375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.19700049700623 - timing_s/old_log_prob:8.195970016000501 - timing_s/adv:0.40654719899612246 - timing_s/update_actor:26.378335635999974 - timing_s/update_weights:1.9244313910021447 - timing_s/step:45.2996021259969 - timing_per_token_ms/adv:0.0027357940216289206 - timing_per_token_ms/update_actor:0.17750876924422773 - timing_per_token_ms/gen:0.07105398175330244 - perf/total_num_tokens:148603 - perf/time_per_step:45.2996021259969 - perf/throughput:3280.4482385225747 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:64 - global_seqlen/min:151310.0 - global_seqlen/max:151310.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151310.0 - global_seqlen/balanced_max:151310.0 - global_seqlen/mean:151310.0 - actor/entropy:0.1377190798521042 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8640551567077637 - training/rollout_probs_diff_mean:0.004494268912822008 - training/rollout_probs_diff_std:0.01500552985817194 - training/rollout_actor_probs_pearson_corr:0.996717095375061 - rollout_corr/training_ppl:1.1572951078414917 - rollout_corr/training_log_ppl:0.1435667872428894 - rollout_corr/kl:0.00101316103246063 - rollout_corr/k3_kl:0.0008953479118645191 - rollout_corr/rollout_ppl:1.1560101509094238 - rollout_corr/rollout_log_ppl:0.14250773191452026 - rollout_corr/log_ppl_diff:0.0010590667370706797 - rollout_corr/log_ppl_abs_diff:0.003194526769220829 - rollout_corr/log_ppl_diff_max:0.03202959895133972 - rollout_corr/log_ppl_diff_min:-0.01776072382926941 - rollout_corr/ppl_ratio:1.0010697841644287 - rollout_corr/chi2_token:0.0014668703079223633 - rollout_corr/chi2_seq:0.1709674596786499 - actor/pg_clipfrac:0.0029563996164749065 - actor/ppo_kl:-0.00019140953626125185 - actor/pg_clipfrac_lower:3.824752305566411e-05 - actor/pg_loss:0.0009195276070386171 - actor/loss:0.0009195266757160425 - actor/grad_norm:1.8017578125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.10421323776245 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:64 - training/epoch:0 - critic/score/mean:0.404296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.404296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006068939808756113 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006068939808756113 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.474609375 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.580078125 - response_length_non_aborted/mean:115.474609375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.580078125 - response/aborted_ratio:0.0 - prompt_length/mean:32.2890625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.208358519994363 - timing_s/old_log_prob:8.358712686000217 - timing_s/adv:0.40694462999817915 - timing_s/update_actor:26.80473640099808 - timing_s/update_weights:1.8848960939940298 - timing_s/step:45.82777920499939 - timing_per_token_ms/adv:0.0026894761086390796 - timing_per_token_ms/update_actor:0.17715112286694917 - timing_per_token_ms/gen:0.0694176422034941 - perf/total_num_tokens:151310 - perf/time_per_step:45.82777920499939 - perf/throughput:3301.709195271096 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:65 - global_seqlen/min:148692.0 - global_seqlen/max:148692.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:148692.0 - global_seqlen/balanced_max:148692.0 - global_seqlen/mean:148692.0 - actor/entropy:0.13634034991264343 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6114726066589355 - training/rollout_probs_diff_mean:0.004436512012034655 - training/rollout_probs_diff_std:0.014783949591219425 - training/rollout_actor_probs_pearson_corr:0.9969331622123718 - rollout_corr/training_ppl:1.1569072008132935 - rollout_corr/training_log_ppl:0.1436890810728073 - rollout_corr/kl:0.000792502542026341 - rollout_corr/k3_kl:0.0008860146044753492 - rollout_corr/rollout_ppl:1.1559252738952637 - rollout_corr/rollout_log_ppl:0.14286449551582336 - rollout_corr/log_ppl_diff:0.0008245707722380757 - rollout_corr/log_ppl_abs_diff:0.0033358093351125717 - rollout_corr/log_ppl_diff_max:0.026237040758132935 - rollout_corr/log_ppl_diff_min:-0.015296012163162231 - rollout_corr/ppl_ratio:1.0008349418640137 - rollout_corr/chi2_token:0.001925349235534668 - rollout_corr/chi2_seq:0.2674522399902344 - actor/pg_clipfrac:0.0031564148084726185 - actor/ppo_kl:-1.696921429091465e-05 - actor/pg_clipfrac_lower:1.8558012016001157e-05 - actor/pg_loss:0.001585624719155021 - actor/loss:0.0015856246463954449 - actor/grad_norm:1.8203125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.203938484191895 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.444) - val-core/char_count/acc/mean@1:np.float64(0.444) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:65 - training/epoch:0 - critic/score/mean:0.4091796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4091796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0080372653901577 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0080372653901577 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:112.79296875 - response_length/max:128.0 - response_length/min:30.0 - response_length/clip_ratio:0.5498046875 - response_length_non_aborted/mean:112.79296875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:30.0 - response_length_non_aborted/clip_ratio:0.5498046875 - response/aborted_ratio:0.0 - prompt_length/mean:32.4140625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.196163704997161 - timing_s/old_log_prob:8.247960999993666 - timing_s/adv:0.18151198099803878 - timing_s/update_actor:26.62079459400411 - timing_s/update_weights:1.917100537000806 - timing_s/step:45.330173645001196 - timing_s/testing:8.6700532429968 - timing_per_token_ms/adv:0.0012207245917604094 - timing_per_token_ms/update_actor:0.17903313287872993 - timing_per_token_ms/gen:0.07096245632032175 - perf/total_num_tokens:148692 - perf/time_per_step:45.330173645001196 - perf/throughput:3280.1992148644035 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 47%|████▋ | 66/140 [53:16<59:02, 47.87s/it] (TaskRunnerV1 pid=2351) Training Progress: 48%|████▊ | 67/140 [54:02<57:32, 47.29s/it] (TaskRunnerV1 pid=2351) Training Progress: 49%|████▊ | 68/140 [54:46<55:40, 46.39s/it] (TaskRunnerV1 pid=2351) step:66 - global_seqlen/min:151258.0 - global_seqlen/max:151258.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151258.0 - global_seqlen/balanced_max:151258.0 - global_seqlen/mean:151258.0 - actor/entropy:0.1396118402481079 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5276423096656799 - training/rollout_probs_diff_mean:0.004591276869177818 - training/rollout_probs_diff_std:0.015547193586826324 - training/rollout_actor_probs_pearson_corr:0.9965283274650574 - rollout_corr/training_ppl:1.1618984937667847 - rollout_corr/training_log_ppl:0.14750893414020538 - rollout_corr/kl:0.0009853626834228635 - rollout_corr/k3_kl:0.001020535477437079 - rollout_corr/rollout_ppl:1.160674810409546 - rollout_corr/rollout_log_ppl:0.14648570120334625 - rollout_corr/log_ppl_diff:0.0010232137283310294 - rollout_corr/log_ppl_abs_diff:0.003535384312272072 - rollout_corr/log_ppl_diff_max:0.04830852150917053 - rollout_corr/log_ppl_diff_min:-0.025753319263458252 - rollout_corr/ppl_ratio:1.001036524772644 - rollout_corr/chi2_token:0.00249481201171875 - rollout_corr/chi2_seq:0.3033177852630615 - actor/pg_clipfrac:0.0029005302931182086 - actor/ppo_kl:-0.00021516263553185885 - actor/pg_clipfrac_lower:2.385190843294064e-05 - actor/pg_loss:-9.058081741386559e-05 - actor/loss:-9.058020077645779e-05 - actor/grad_norm:1.751953125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.15356254577637 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:66 - training/epoch:0 - critic/score/mean:0.4150390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4150390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007534346077591181 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007534346077591181 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:114.775390625 - response_length/max:128.0 - response_length/min:29.0 - response_length/clip_ratio:0.57421875 - response_length_non_aborted/mean:114.775390625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:29.0 - response_length_non_aborted/clip_ratio:0.57421875 - response/aborted_ratio:0.0 - prompt_length/mean:32.9375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.207370002004609 - timing_s/old_log_prob:8.631370477996825 - timing_s/adv:0.2241996089942404 - timing_s/update_actor:26.698457613994833 - timing_s/update_weights:1.8661979049938964 - timing_s/step:45.79736929399951 - timing_per_token_ms/adv:0.0014822330653204486 - timing_per_token_ms/update_actor:0.1765093919924555 - timing_per_token_ms/gen:0.06983212798438364 - perf/total_num_tokens:151258 - perf/time_per_step:45.79736929399951 - perf/throughput:3302.766126783143 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:67 - global_seqlen/min:151839.0 - global_seqlen/max:151839.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151839.0 - global_seqlen/balanced_max:151839.0 - global_seqlen/mean:151839.0 - actor/entropy:0.1378505527973175 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5330614447593689 - training/rollout_probs_diff_mean:0.004436471965163946 - training/rollout_probs_diff_std:0.01492022629827261 - training/rollout_actor_probs_pearson_corr:0.9968229532241821 - rollout_corr/training_ppl:1.1600421667099 - rollout_corr/training_log_ppl:0.14593923091888428 - rollout_corr/kl:0.0008046072907745838 - rollout_corr/k3_kl:0.0008984992746263742 - rollout_corr/rollout_ppl:1.1590534448623657 - rollout_corr/rollout_log_ppl:0.1451309323310852 - rollout_corr/log_ppl_diff:0.0008083049324341118 - rollout_corr/log_ppl_abs_diff:0.0032577463425695896 - rollout_corr/log_ppl_diff_max:0.03171224892139435 - rollout_corr/log_ppl_diff_min:-0.02547392249107361 - rollout_corr/ppl_ratio:1.0008183717727661 - rollout_corr/chi2_token:0.0020743608474731445 - rollout_corr/chi2_seq:0.2879892587661743 - actor/pg_clipfrac:0.003375471977051347 - actor/ppo_kl:-4.2399500140769916e-05 - actor/pg_clipfrac_lower:7.57025190978311e-06 - actor/pg_loss:0.0007087434642016888 - actor/loss:0.0007087441626936197 - actor/grad_norm:2.04296875 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.11383104324341 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:67 - training/epoch:0 - critic/score/mean:0.388671875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.388671875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0050656236708164215 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0050656236708164215 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:115.4365234375 - response_length/max:128.0 - response_length/min:45.0 - response_length/clip_ratio:0.583984375 - response_length_non_aborted/mean:115.4365234375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:45.0 - response_length_non_aborted/clip_ratio:0.583984375 - response/aborted_ratio:0.0 - prompt_length/mean:32.84375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0234375 - timing_s/gen:8.19874044899916 - timing_s/old_log_prob:8.361107514007017 - timing_s/adv:0.3798012719998951 - timing_s/update_actor:26.814530120995187 - timing_s/update_weights:1.8964395099974354 - timing_s/step:45.81933550300164 - timing_per_token_ms/adv:0.0025013420267513296 - timing_per_token_ms/update_actor:0.17659843729868602 - timing_per_token_ms/gen:0.06935917880497061 - perf/total_num_tokens:151839 - perf/time_per_step:45.81933550300164 - perf/throughput:3313.8629867308523 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:68 - global_seqlen/min:144726.0 - global_seqlen/max:144726.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:144726.0 - global_seqlen/balanced_max:144726.0 - global_seqlen/mean:144726.0 - actor/entropy:0.14509433507919312 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.944478452205658 - training/rollout_probs_diff_mean:0.004784837830811739 - training/rollout_probs_diff_std:0.01588035747408867 - training/rollout_actor_probs_pearson_corr:0.9965057969093323 - rollout_corr/training_ppl:1.166192650794983 - rollout_corr/training_log_ppl:0.1512405276298523 - rollout_corr/kl:0.0010717058321461082 - rollout_corr/k3_kl:0.0009970961837098002 - rollout_corr/rollout_ppl:1.164828896522522 - rollout_corr/rollout_log_ppl:0.15012525022029877 - rollout_corr/log_ppl_diff:0.001115265884436667 - rollout_corr/log_ppl_abs_diff:0.0035839825868606567 - rollout_corr/log_ppl_diff_max:0.05547322332859039 - rollout_corr/log_ppl_diff_min:-0.023853003978729248 - rollout_corr/ppl_ratio:1.0011301040649414 - rollout_corr/chi2_token:0.0017718076705932617 - rollout_corr/chi2_seq:0.24689579010009766 - actor/pg_clipfrac:0.0031342045064472286 - actor/ppo_kl:-0.00010968620275283077 - actor/pg_clipfrac_lower:1.998994866880821e-05 - actor/pg_loss:0.000645540829282254 - actor/loss:0.0006455411203205585 - actor/grad_norm:1.6943359375 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.225446701049805 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:68 - training/epoch:0 - critic/score/mean:0.455078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.455078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0023986303713172674 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0023986303713172674 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.373046875 - response_length/max:128.0 - response_length/min:28.0 - response_length/clip_ratio:0.470703125 - response_length_non_aborted/mean:109.373046875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:28.0 - response_length_non_aborted/clip_ratio:0.470703125 - response/aborted_ratio:0.0 - prompt_length/mean:31.9609375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.191192822996527 - timing_s/old_log_prob:8.027753517002566 - timing_s/adv:0.3868830749997869 - timing_s/update_actor:25.504063749001944 - timing_s/update_weights:1.879474287998164 - timing_s/step:44.17024686399964 - timing_per_token_ms/adv:0.0026732105841368306 - timing_per_token_ms/update_actor:0.17622309570500078 - timing_per_token_ms/gen:0.07313695622240153 - perf/total_num_tokens:144726 - perf/time_per_step:44.17024686399964 - perf/throughput:3276.549493726216 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TransferQueueController pid=2438) 2026-11-01 18:11:03,460 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 32276, Total req/min: 5919.88, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.45 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=12035, req/min=2207.39, avg_time=0.000259s, max_time=0.070395s, min_time=0.000064s; CLEAR_META: req_count=16, req/min=2.93, avg_time=0.001585s, max_time=0.003837s, min_time=0.000775s; SET_CUSTOM_META: req_count=11989, req/min=2198.95, avg_time=0.000009s, max_time=0.000243s, min_time=0.000003s; KV_LIST: req_count=40, req/min=7.34, avg_time=0.000202s, max_time=0.000741s, min_time=0.000054s; NOTIFY_DATA_UPDATE: req_count=8189, req/min=1501.98, avg_time=0.000081s, max_time=0.001203s, min_time=0.000031s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.28, avg_time=0.000126s, max_time=0.000213s, min_time=0.000105s (TaskRunnerV1 pid=2351) Training Progress: 49%|████▉ | 69/140 [55:32<54:36, 46.15s/it] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=2.90, avg_time=0.001517s, max_time=0.004131s, min_time=0.000410s; PUT_DATA: req_count=1036, req/min=188.00, avg_time=0.000594s, max_time=0.001302s, min_time=0.000311s; GET_DATA: req_count=36, req/min=6.53, avg_time=0.000597s, max_time=0.001209s, min_time=0.000398s (SimpleStorageUnit pid=2583) 2026-11-01 18:11:52,460 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1088, Total req/min: 197.44, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2583) Time range: last 5.51 minutes; (SimpleStorageUnit pid=2596) 2026-11-01 18:11:52,563 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_72b4f821: [Performance] Total success requests: 1105, Total req/min: 200.46, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2596) Time range: last 5.51 minutes; (TaskRunnerV1 pid=2351) Training Progress: 50%|█████ | 70/140 [56:26<56:43, 48.62s/it] (SimpleStorageUnit pid=2584) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=2.90, avg_time=0.001536s, max_time=0.004465s, min_time=0.000411s; PUT_DATA: req_count=1017, req/min=184.37, avg_time=0.000592s, max_time=0.002786s, min_time=0.000315s; GET_DATA: req_count=36, req/min=6.53, avg_time=0.000627s, max_time=0.001274s, min_time=0.000399s [repeated 7x across cluster] (SimpleStorageUnit pid=2584) 2026-11-01 18:11:52,784 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_fabcdd0d: [Performance] Total success requests: 1069, Total req/min: 193.80, Total avg process time: 0.0006s; [repeated 6x across cluster] (SimpleStorageUnit pid=2584) Time range: last 5.52 minutes; [repeated 6x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 51%|█████ | 71/140 [57:12<55:00, 47.83s/it] (TaskRunnerV1 pid=2351) step:69 - global_seqlen/min:150109.0 - global_seqlen/max:150109.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:150109.0 - global_seqlen/balanced_max:150109.0 - global_seqlen/mean:150109.0 - actor/entropy:0.13892343640327454 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.41412869095802307 - training/rollout_probs_diff_mean:0.004549822304397821 - training/rollout_probs_diff_std:0.01452714391052723 - training/rollout_actor_probs_pearson_corr:0.9969745874404907 - rollout_corr/training_ppl:1.1585054397583008 - rollout_corr/training_log_ppl:0.14477239549160004 - rollout_corr/kl:0.0008338991901837289 - rollout_corr/k3_kl:0.0008925475995056331 - rollout_corr/rollout_ppl:1.1573915481567383 - rollout_corr/rollout_log_ppl:0.14384505152702332 - rollout_corr/log_ppl_diff:0.0009273559553548694 - rollout_corr/log_ppl_abs_diff:0.0033621618058532476 - rollout_corr/log_ppl_diff_max:0.02718275785446167 - rollout_corr/log_ppl_diff_min:-0.040869295597076416 - rollout_corr/ppl_ratio:1.0009382963180542 - rollout_corr/chi2_token:0.0020426511764526367 - rollout_corr/chi2_seq:0.6167796850204468 - actor/pg_clipfrac:0.003315324807772413 - actor/ppo_kl:5.539910227980728e-05 - actor/pg_clipfrac_lower:7.578513153324214e-06 - actor/pg_loss:0.002292889083037153 - actor/loss:0.0022928868420422077 - actor/grad_norm:1.9404296875 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.287109375 - actor/perf/cpu_memory_used_gb:42.103782176971436 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:69 - training/epoch:0 - critic/score/mean:0.4453125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4453125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008852394297719002 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008852394297719002 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:113.7939453125 - response_length/max:128.0 - response_length/min:35.0 - response_length/clip_ratio:0.52734375 - response_length_non_aborted/mean:113.7939453125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:35.0 - response_length_non_aborted/clip_ratio:0.52734375 - response/aborted_ratio:0.0 - prompt_length/mean:32.796875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.199786855999264 - timing_s/old_log_prob:8.262052230005793 - timing_s/adv:0.18268596999405418 - timing_s/update_actor:26.573343825002667 - timing_s/update_weights:1.8901819109960343 - timing_s/step:45.27797616300086 - timing_per_token_ms/adv:0.0012170220972363695 - timing_per_token_ms/update_actor:0.17702698589027085 - timing_per_token_ms/gen:0.07036933581634211 - perf/total_num_tokens:150109 - perf/time_per_step:45.27797616300086 - perf/throughput:3315.2762716161856 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:70 - global_seqlen/min:150216.0 - global_seqlen/max:150216.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:150216.0 - global_seqlen/balanced_max:150216.0 - global_seqlen/mean:150216.0 - actor/entropy:0.1411801278591156 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9804071187973022 - training/rollout_probs_diff_mean:0.004567216150462627 - training/rollout_probs_diff_std:0.015260686166584492 - training/rollout_actor_probs_pearson_corr:0.9967670440673828 - rollout_corr/training_ppl:1.1622384786605835 - rollout_corr/training_log_ppl:0.14777234196662903 - rollout_corr/kl:0.0010068458504974842 - rollout_corr/k3_kl:0.0009692308958619833 - rollout_corr/rollout_ppl:1.1609834432601929 - rollout_corr/rollout_log_ppl:0.14673474431037903 - rollout_corr/log_ppl_diff:0.0010376193094998598 - rollout_corr/log_ppl_abs_diff:0.0032557218801230192 - rollout_corr/log_ppl_diff_max:0.07266402244567871 - rollout_corr/log_ppl_diff_min:-0.019793957471847534 - rollout_corr/ppl_ratio:1.0010499954223633 - rollout_corr/chi2_token:0.0016654729843139648 - rollout_corr/chi2_seq:0.21707749366760254 - actor/pg_clipfrac:0.0037986234965501357 - actor/ppo_kl:8.583303345706853e-06 - actor/pg_clipfrac_lower:2.5377063745205913e-05 - actor/pg_loss:-2.7882473659701645e-05 - actor/loss:-2.7882633730769157e-05 - actor/grad_norm:1.8759765625 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.20931911468506 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.452) - val-core/char_count/acc/mean@1:np.float64(0.452) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:70 - training/epoch:0 - critic/score/mean:0.4755859375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4755859375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00359171605668962 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.00359171605668962 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:114.3671875 - response_length/max:128.0 - response_length/min:34.0 - response_length/clip_ratio:0.5244140625 - response_length_non_aborted/mean:114.3671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:34.0 - response_length_non_aborted/clip_ratio:0.5244140625 - response/aborted_ratio:0.0 - prompt_length/mean:32.328125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.180515554005979 - timing_s/old_log_prob:8.255359183996916 - timing_s/adv:0.18351053399965167 - timing_s/update_actor:26.636762631998863 - timing_s/update_weights:1.8702822890045354 - timing_s/step:45.29763072300557 - timing_s/testing:8.962675037997542 - timing_per_token_ms/adv:0.0012216443920730926 - timing_per_token_ms/update_actor:0.17732307232251468 - timing_per_token_ms/gen:0.06985206942077651 - perf/total_num_tokens:150216 - perf/time_per_step:45.29763072300557 - perf/throughput:3316.1999336912104 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:71 - global_seqlen/min:150214.0 - global_seqlen/max:150214.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:150214.0 - global_seqlen/balanced_max:150214.0 - global_seqlen/mean:150214.0 - actor/entropy:0.1374877393245697 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9702568650245667 - training/rollout_probs_diff_mean:0.0045867422595620155 - training/rollout_probs_diff_std:0.015734495595097542 - training/rollout_actor_probs_pearson_corr:0.9964178204536438 - rollout_corr/training_ppl:1.1574710607528687 - rollout_corr/training_log_ppl:0.14397922158241272 - rollout_corr/kl:0.0007684130687266588 - rollout_corr/k3_kl:0.00120161019731313 - rollout_corr/rollout_ppl:1.1566548347473145 - rollout_corr/rollout_log_ppl:0.14328519999980927 - rollout_corr/log_ppl_diff:0.000694009882863611 - rollout_corr/log_ppl_abs_diff:0.0032750852406024933 - rollout_corr/log_ppl_diff_max:0.04048939049243927 - rollout_corr/log_ppl_diff_min:-0.05136553943157196 - rollout_corr/ppl_ratio:1.0007054805755615 - rollout_corr/chi2_token:0.012224078178405762 - rollout_corr/chi2_seq:1.1194736957550049 - actor/pg_clipfrac:0.003683738182492865 - actor/ppo_kl:0.0003474871867486703 - actor/pg_clipfrac_lower:8.612373979606977e-06 - actor/pg_loss:0.001548808068037033 - actor/loss:0.0015488066710531712 - actor/grad_norm:1.9130859375 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.20508909225464 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:71 - training/epoch:1 - critic/score/mean:0.439453125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.439453125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0010529974242672324 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0010529974242672324 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:114.271484375 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.5361328125 - response_length_non_aborted/mean:114.271484375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.5361328125 - response/aborted_ratio:0.0 - prompt_length/mean:32.421875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:8.21023368799797 - timing_s/old_log_prob:8.296609069002443 - timing_s/adv:0.43776894999609794 - timing_s/update_actor:26.6421233880028 - timing_s/update_weights:1.9091862730056164 - timing_s/step:45.851435365002544 - timing_per_token_ms/adv:0.00291430192922163 - timing_per_token_ms/update_actor:0.1773611207211232 - timing_per_token_ms/gen:0.07016454174712403 - perf/total_num_tokens:150214 - perf/time_per_step:45.851435365002544 - perf/throughput:3276.10245577296 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 51%|█████▏ | 72/140 [57:57<53:08, 46.89s/it] (TaskRunnerV1 pid=2351) Training Progress: 52%|█████▏ | 73/140 [58:42<51:49, 46.42s/it] (TaskRunnerV1 pid=2351) Training Progress: 53%|█████▎ | 74/140 [59:28<50:40, 46.07s/it] (TaskRunnerV1 pid=2351) step:72 - global_seqlen/min:146928.0 - global_seqlen/max:146928.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146928.0 - global_seqlen/balanced_max:146928.0 - global_seqlen/mean:146928.0 - actor/entropy:0.14389289915561676 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7243660688400269 - training/rollout_probs_diff_mean:0.0047509740106761456 - training/rollout_probs_diff_std:0.015748174861073494 - training/rollout_actor_probs_pearson_corr:0.9965062737464905 - rollout_corr/training_ppl:1.1682690382003784 - rollout_corr/training_log_ppl:0.1527586281299591 - rollout_corr/kl:0.0009169129771180451 - rollout_corr/k3_kl:0.0010530557483434677 - rollout_corr/rollout_ppl:1.1671297550201416 - rollout_corr/rollout_log_ppl:0.15181852877140045 - rollout_corr/log_ppl_diff:0.0009400981944054365 - rollout_corr/log_ppl_abs_diff:0.0034504139330238104 - rollout_corr/log_ppl_diff_max:0.029825372621417046 - rollout_corr/log_ppl_diff_min:-0.016551733016967773 - rollout_corr/ppl_ratio:1.0009509325027466 - rollout_corr/chi2_token:0.002552628517150879 - rollout_corr/chi2_seq:0.2563279867172241 - actor/pg_clipfrac:0.0032927866025905432 - actor/ppo_kl:4.296611386204555e-05 - actor/pg_clipfrac_lower:2.3444206817657687e-05 - actor/pg_loss:0.0011386778205633163 - actor/loss:0.001138677354902029 - actor/grad_norm:1.76953125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.1390962600708 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:72 - training/epoch:1 - critic/score/mean:0.458984375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.458984375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005700853653252125 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005700853653252125 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:111.2734375 - response_length/max:128.0 - response_length/min:33.0 - response_length/clip_ratio:0.5009765625 - response_length_non_aborted/mean:111.2734375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:33.0 - response_length_non_aborted/clip_ratio:0.5009765625 - response/aborted_ratio:0.0 - prompt_length/mean:32.2109375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.197319512000831 - timing_s/old_log_prob:8.165006542003539 - timing_s/adv:0.1832253259999561 - timing_s/update_actor:25.983741147996625 - timing_s/update_weights:1.8692653990001418 - timing_s/step:44.564627965999534 - timing_per_token_ms/adv:0.001247041584993712 - timing_per_token_ms/update_actor:0.17684676268646293 - timing_per_token_ms/gen:0.07194165126729649 - perf/total_num_tokens:146928 - perf/time_per_step:44.564627965999534 - perf/throughput:3296.964581687933 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:73 - global_seqlen/min:148243.0 - global_seqlen/max:148243.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:148243.0 - global_seqlen/balanced_max:148243.0 - global_seqlen/mean:148243.0 - actor/entropy:0.1411036103963852 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9061552286148071 - training/rollout_probs_diff_mean:0.00460008392110467 - training/rollout_probs_diff_std:0.01582128368318081 - training/rollout_actor_probs_pearson_corr:0.996520459651947 - rollout_corr/training_ppl:1.1617445945739746 - rollout_corr/training_log_ppl:0.1478319764137268 - rollout_corr/kl:0.0009558399906381965 - rollout_corr/k3_kl:0.0009508530492894351 - rollout_corr/rollout_ppl:1.160606861114502 - rollout_corr/rollout_log_ppl:0.14687711000442505 - rollout_corr/log_ppl_diff:0.0009548537200316787 - rollout_corr/log_ppl_abs_diff:0.003287313040345907 - rollout_corr/log_ppl_diff_max:0.04968036711215973 - rollout_corr/log_ppl_diff_min:-0.015057384967803955 - rollout_corr/ppl_ratio:1.0009655952453613 - rollout_corr/chi2_token:0.0017647743225097656 - rollout_corr/chi2_seq:0.20323896408081055 - actor/pg_clipfrac:0.0035670142776022358 - actor/ppo_kl:0.0002453954402881209 - actor/pg_clipfrac_lower:7.798365307583785e-06 - actor/pg_loss:0.00305264035705477 - actor/loss:0.003052642336115241 - actor/grad_norm:1.734375 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.20800161361694 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:73 - training/epoch:1 - critic/score/mean:0.4560546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4560546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.010019603185355663 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.010019603185355663 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:112.3232421875 - response_length/max:128.0 - response_length/min:34.0 - response_length/clip_ratio:0.49609375 - response_length_non_aborted/mean:112.3232421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:34.0 - response_length_non_aborted/clip_ratio:0.49609375 - response/aborted_ratio:0.0 - prompt_length/mean:32.4453125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.194992220000131 - timing_s/old_log_prob:8.282291432995407 - timing_s/adv:0.18427404099929845 - timing_s/update_actor:26.301572547999967 - timing_s/update_weights:1.8769830640012515 - timing_s/step:45.00398319699889 - timing_per_token_ms/adv:0.0012430539114784405 - timing_per_token_ms/update_actor:0.17742202025053438 - timing_per_token_ms/gen:0.07124903033411985 - perf/total_num_tokens:148243 - perf/time_per_step:45.00398319699889 - perf/throughput:3293.997319105871 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:74 - global_seqlen/min:147515.0 - global_seqlen/max:147515.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147515.0 - global_seqlen/balanced_max:147515.0 - global_seqlen/mean:147515.0 - actor/entropy:0.1399001181125641 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7802472114562988 - training/rollout_probs_diff_mean:0.004672458861023188 - training/rollout_probs_diff_std:0.015567800030112267 - training/rollout_actor_probs_pearson_corr:0.9965679049491882 - rollout_corr/training_ppl:1.1625012159347534 - rollout_corr/training_log_ppl:0.14817918837070465 - rollout_corr/kl:0.001018706476315856 - rollout_corr/k3_kl:0.000942396407481283 - rollout_corr/rollout_ppl:1.1612130403518677 - rollout_corr/rollout_log_ppl:0.14711329340934753 - rollout_corr/log_ppl_diff:0.0010658788960427046 - rollout_corr/log_ppl_abs_diff:0.003455652389675379 - rollout_corr/log_ppl_diff_max:0.02114538848400116 - rollout_corr/log_ppl_diff_min:-0.027622327208518982 - rollout_corr/ppl_ratio:1.0010768175125122 - rollout_corr/chi2_token:0.0016793012619018555 - rollout_corr/chi2_seq:0.18629086017608643 - actor/pg_clipfrac:0.003820381990711515 - actor/ppo_kl:0.0001384390369215301 - actor/pg_clipfrac_lower:7.5771349656861275e-06 - actor/pg_loss:0.0008685133070684969 - actor/loss:0.000868513947352767 - actor/grad_norm:1.9501953125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.14075994491577 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:74 - training/epoch:1 - critic/score/mean:0.4306640625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4306640625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0062957750633358955 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0062957750633358955 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:111.4482421875 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.50390625 - response_length_non_aborted/mean:111.4482421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.50390625 - response/aborted_ratio:0.0 - prompt_length/mean:32.609375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.1015625 - timing_s/gen:8.202273807000893 - timing_s/old_log_prob:8.288488466998388 - timing_s/adv:0.22474681500170846 - timing_s/update_actor:26.331636810005875 - timing_s/update_weights:1.909654353003134 - timing_s/step:45.122322215996974 - timing_per_token_ms/adv:0.001523552282830278 - timing_per_token_ms/update_actor:0.1785014189065917 - timing_per_token_ms/gen:0.07187222389002122 - perf/total_num_tokens:147515 - perf/time_per_step:45.122322215996974 - perf/throughput:3269.2244715122906 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 54%|█████▎ | 75/140 [1:00:21<52:16, 48.25s/it] (TransferQueueController pid=2438) 2026-11-01 18:16:09,003 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 39277, Total req/min: 7712.90, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.09 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=15034, req/min=2952.25, avg_time=0.000225s, max_time=0.017475s, min_time=0.000063s; CLEAR_META: req_count=18, req/min=3.53, avg_time=0.001679s, max_time=0.003322s, min_time=0.000872s; SET_CUSTOM_META: req_count=14987, req/min=2943.02, avg_time=0.000009s, max_time=0.000138s, min_time=0.000003s; KV_LIST: req_count=45, req/min=8.84, avg_time=0.000238s, max_time=0.000751s, min_time=0.000047s; NOTIFY_DATA_UPDATE: req_count=9187, req/min=1804.07, avg_time=0.000081s, max_time=0.000717s, min_time=0.000031s; KV_RETRIEVE_KEYS: req_count=6, req/min=1.18, avg_time=0.000121s, max_time=0.000144s, min_time=0.000111s (TaskRunnerV1 pid=2351) Training Progress: 54%|█████▍ | 76/140 [1:01:06<50:22, 47.22s/it] (SimpleStorageUnit pid=2599) 2026-11-01 18:16:54,049 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1212, Total req/min: 241.31, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.02 minutes; (SimpleStorageUnit pid=2599) Per-operation statistics: PUT_DATA: req_count=1158, req/min=230.56, avg_time=0.000611s, max_time=0.001593s, min_time=0.000318s; CLEAR_DATA: req_count=18, req/min=3.58, avg_time=0.001567s, max_time=0.003886s, min_time=0.000396s; GET_DATA: req_count=36, req/min=7.17, avg_time=0.000642s, max_time=0.001250s, min_time=0.000412s (TaskRunnerV1 pid=2351) Training Progress: 55%|█████▌ | 77/140 [1:01:52<49:11, 46.86s/it] (SimpleStorageUnit pid=2583) 2026-11-01 18:16:54,055 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1211, Total req/min: 240.92, Total avg process time: 0.0006s; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.03 minutes; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: PUT_DATA: req_count=1157, req/min=230.18, avg_time=0.000597s, max_time=0.001713s, min_time=0.000320s; CLEAR_DATA: req_count=18, req/min=3.58, avg_time=0.001267s, max_time=0.003635s, min_time=0.000411s; GET_DATA: req_count=36, req/min=7.16, avg_time=0.000594s, max_time=0.001221s, min_time=0.000367s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) step:75 - global_seqlen/min:146655.0 - global_seqlen/max:146655.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146655.0 - global_seqlen/balanced_max:146655.0 - global_seqlen/mean:146655.0 - actor/entropy:0.1393253654241562 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8378893136978149 - training/rollout_probs_diff_mean:0.004678565543144941 - training/rollout_probs_diff_std:0.01598631963133812 - training/rollout_actor_probs_pearson_corr:0.996380627155304 - rollout_corr/training_ppl:1.161453366279602 - rollout_corr/training_log_ppl:0.14716628193855286 - rollout_corr/kl:0.001081398455426097 - rollout_corr/k3_kl:0.0009972024708986282 - rollout_corr/rollout_ppl:1.160139799118042 - rollout_corr/rollout_log_ppl:0.14606526494026184 - rollout_corr/log_ppl_diff:0.0011010258458554745 - rollout_corr/log_ppl_abs_diff:0.003638964146375656 - rollout_corr/log_ppl_diff_max:0.027315661311149597 - rollout_corr/log_ppl_diff_min:-0.016103357076644897 - rollout_corr/ppl_ratio:1.0011134147644043 - rollout_corr/chi2_token:0.0016431808471679688 - rollout_corr/chi2_seq:0.23185133934020996 - actor/pg_clipfrac:0.0030462539677197737 - actor/ppo_kl:-0.00012303721640212945 - actor/pg_clipfrac_lower:3.991307433655796e-05 - actor/pg_loss:-0.00015194760635495186 - actor/loss:-0.00015194574370980263 - actor/grad_norm:1.767578125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.199461460113525 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.474) - val-core/char_count/acc/mean@1:np.float64(0.474) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:75 - training/epoch:1 - critic/score/mean:0.4052734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4052734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00563014717772603 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.00563014717772603 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:110.6083984375 - response_length/max:128.0 - response_length/min:40.0 - response_length/clip_ratio:0.498046875 - response_length_non_aborted/mean:110.6083984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:40.0 - response_length_non_aborted/clip_ratio:0.498046875 - response/aborted_ratio:0.0 - prompt_length/mean:32.609375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.185558434000995 - timing_s/old_log_prob:8.141792686001281 - timing_s/adv:0.18762429800699465 - timing_s/update_actor:25.76455473600072 - timing_s/update_weights:1.8846378669986734 - timing_s/step:44.333323045000725 - timing_s/testing:8.882397649998893 - timing_per_token_ms/adv:0.0012793583444614548 - timing_per_token_ms/update_actor:0.17568139331083643 - timing_per_token_ms/gen:0.07227036573286065 - perf/total_num_tokens:146655 - perf/time_per_step:44.333323045000725 - perf/throughput:3308.008286478711 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:76 - global_seqlen/min:146628.0 - global_seqlen/max:146628.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146628.0 - global_seqlen/balanced_max:146628.0 - global_seqlen/mean:146628.0 - actor/entropy:0.1359901875257492 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9522387981414795 - training/rollout_probs_diff_mean:0.0044806525111198425 - training/rollout_probs_diff_std:0.015176265500485897 - training/rollout_actor_probs_pearson_corr:0.9966673254966736 - rollout_corr/training_ppl:1.1590800285339355 - rollout_corr/training_log_ppl:0.14516659080982208 - rollout_corr/kl:0.0009455645922571421 - rollout_corr/k3_kl:0.0009472475503571332 - rollout_corr/rollout_ppl:1.1579346656799316 - rollout_corr/rollout_log_ppl:0.1442146599292755 - rollout_corr/log_ppl_diff:0.0009519282612018287 - rollout_corr/log_ppl_abs_diff:0.0033886379096657038 - rollout_corr/log_ppl_diff_max:0.07405281066894531 - rollout_corr/log_ppl_diff_min:-0.01676642894744873 - rollout_corr/ppl_ratio:1.000964641571045 - rollout_corr/chi2_token:0.001740097999572754 - rollout_corr/chi2_seq:0.1956648826599121 - actor/pg_clipfrac:0.0036462187563301995 - actor/ppo_kl:-9.702505972578972e-05 - actor/pg_clipfrac_lower:7.593706565482232e-06 - actor/pg_loss:0.00249426846858114 - actor/loss:0.0024942702148109674 - actor/grad_norm:2.0791015625 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.15053415298462 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:76 - training/epoch:1 - critic/score/mean:0.48828125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.48828125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008302479982376099 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008302479982376099 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:110.63671875 - response_length/max:128.0 - response_length/min:32.0 - response_length/clip_ratio:0.462890625 - response_length_non_aborted/mean:110.63671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:32.0 - response_length_non_aborted/clip_ratio:0.462890625 - response/aborted_ratio:0.0 - prompt_length/mean:32.5546875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.209015340995393 - timing_s/old_log_prob:8.09169539999857 - timing_s/adv:0.19755980699846987 - timing_s/update_actor:26.08306456900027 - timing_s/update_weights:1.8799281810061075 - timing_s/step:44.63119033699331 - timing_per_token_ms/adv:0.0013473538955620335 - timing_per_token_ms/update_actor:0.17788597381809934 - timing_per_token_ms/gen:0.07245891449524584 - perf/total_num_tokens:146628 - perf/time_per_step:44.63119033699331 - perf/throughput:3285.3257753796656 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:77 - global_seqlen/min:151185.0 - global_seqlen/max:151185.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:151185.0 - global_seqlen/balanced_max:151185.0 - global_seqlen/mean:151185.0 - actor/entropy:0.13397887349128723 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9241016507148743 - training/rollout_probs_diff_mean:0.004475610330700874 - training/rollout_probs_diff_std:0.016234811395406723 - training/rollout_actor_probs_pearson_corr:0.9960905313491821 - rollout_corr/training_ppl:1.1535732746124268 - rollout_corr/training_log_ppl:0.14068515598773956 - rollout_corr/kl:0.0010159931844100356 - rollout_corr/k3_kl:0.0010136117925867438 - rollout_corr/rollout_ppl:1.1524672508239746 - rollout_corr/rollout_log_ppl:0.13973408937454224 - rollout_corr/log_ppl_diff:0.0009510587551631033 - rollout_corr/log_ppl_abs_diff:0.0033444552682340145 - rollout_corr/log_ppl_diff_max:0.03260618448257446 - rollout_corr/log_ppl_diff_min:-0.018990367650985718 - rollout_corr/ppl_ratio:1.0009623765945435 - rollout_corr/chi2_token:0.0019905567169189453 - rollout_corr/chi2_seq:0.23466408252716064 - actor/pg_clipfrac:0.0037409131376383207 - actor/ppo_kl:-7.388628612640485e-05 - actor/pg_clipfrac_lower:4.1037646951735944e-05 - actor/pg_loss:0.0009155541192740202 - actor/loss:0.0009155538864433765 - actor/grad_norm:1.87109375 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.152554512023926 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:77 - training/epoch:1 - critic/score/mean:0.3984375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3984375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0011737033491954207 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0011737033491954207 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:114.5087890625 - response_length/max:128.0 - response_length/min:33.0 - response_length/clip_ratio:0.56640625 - response_length_non_aborted/mean:114.5087890625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:33.0 - response_length_non_aborted/clip_ratio:0.56640625 - response/aborted_ratio:0.0 - prompt_length/mean:33.1328125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.199608577997424 - timing_s/old_log_prob:8.336887536999711 - timing_s/adv:0.19869508199917618 - timing_s/update_actor:26.92103807899548 - timing_s/update_weights:1.8801691539993044 - timing_s/step:45.70330722499784 - timing_per_token_ms/adv:0.0013142512947658576 - timing_per_token_ms/update_actor:0.17806685900714675 - timing_per_token_ms/gen:0.06992852092410196 - perf/total_num_tokens:151185 - perf/time_per_step:45.70330722499784 - perf/throughput:3307.966297837369 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 56%|█████▌ | 78/140 [1:02:36<47:33, 46.03s/it] (TaskRunnerV1 pid=2351) Training Progress: 56%|█████▋ | 79/140 [1:03:20<46:19, 45.56s/it] (TaskRunnerV1 pid=2351) Training Progress: 57%|█████▋ | 80/140 [1:04:15<48:15, 48.26s/it] (TaskRunnerV1 pid=2351) step:78 - global_seqlen/min:143892.0 - global_seqlen/max:143892.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:143892.0 - global_seqlen/balanced_max:143892.0 - global_seqlen/mean:143892.0 - actor/entropy:0.14017130434513092 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9234321713447571 - training/rollout_probs_diff_mean:0.004721052944660187 - training/rollout_probs_diff_std:0.01627243496477604 - training/rollout_actor_probs_pearson_corr:0.9962009787559509 - rollout_corr/training_ppl:1.163880705833435 - rollout_corr/training_log_ppl:0.14927047491073608 - rollout_corr/kl:0.0010117116617038846 - rollout_corr/k3_kl:0.0010556795168668032 - rollout_corr/rollout_ppl:1.1626849174499512 - rollout_corr/rollout_log_ppl:0.14827275276184082 - rollout_corr/log_ppl_diff:0.0009977242443710566 - rollout_corr/log_ppl_abs_diff:0.0034851122181862593 - rollout_corr/log_ppl_diff_max:0.05482184886932373 - rollout_corr/log_ppl_diff_min:-0.021563127636909485 - rollout_corr/ppl_ratio:1.001010537147522 - rollout_corr/chi2_token:0.001973390579223633 - rollout_corr/chi2_seq:0.20474207401275635 - actor/pg_clipfrac:0.0035239382850704715 - actor/ppo_kl:-2.049559005475506e-05 - actor/pg_clipfrac_lower:2.4280816433019936e-05 - actor/pg_loss:0.002997348718054127 - actor/loss:0.0029973487253300846 - actor/grad_norm:1.96484375 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.29469537734985 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:78 - training/epoch:1 - critic/score/mean:0.45703125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.45703125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.009913401678204536 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.009913401678204536 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:108.60546875 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.44140625 - response_length_non_aborted/mean:108.60546875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.44140625 - response/aborted_ratio:0.0 - prompt_length/mean:31.9140625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.192908788005298 - timing_s/old_log_prob:8.078167505002057 - timing_s/adv:0.18618387300375616 - timing_s/update_actor:25.441516342994873 - timing_s/update_weights:1.8994504550064448 - timing_s/step:43.97000383599516 - timing_per_token_ms/adv:0.001293913998024603 - timing_per_token_ms/update_actor:0.1768098041794879 - timing_per_token_ms/gen:0.07366928737910745 - perf/total_num_tokens:143892 - perf/time_per_step:43.97000383599516 - perf/throughput:3272.5036944892354 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:79 - global_seqlen/min:145552.0 - global_seqlen/max:145552.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:145552.0 - global_seqlen/balanced_max:145552.0 - global_seqlen/mean:145552.0 - actor/entropy:0.13922421634197235 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5328643321990967 - training/rollout_probs_diff_mean:0.004614867269992828 - training/rollout_probs_diff_std:0.015310017392039299 - training/rollout_actor_probs_pearson_corr:0.9966235756874084 - rollout_corr/training_ppl:1.1623231172561646 - rollout_corr/training_log_ppl:0.1478334367275238 - rollout_corr/kl:0.0011593116214498878 - rollout_corr/k3_kl:0.0010696093086153269 - rollout_corr/rollout_ppl:1.1608227491378784 - rollout_corr/rollout_log_ppl:0.1466204822063446 - rollout_corr/log_ppl_diff:0.0012129542883485556 - rollout_corr/log_ppl_abs_diff:0.0038277730345726013 - rollout_corr/log_ppl_diff_max:0.07129138708114624 - rollout_corr/log_ppl_diff_min:-0.019689783453941345 - rollout_corr/ppl_ratio:1.001230001449585 - rollout_corr/chi2_token:0.0017049312591552734 - rollout_corr/chi2_seq:0.23757290840148926 - actor/pg_clipfrac:0.003326415113406256 - actor/ppo_kl:0.00010413535876333906 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0006404972159543831 - actor/loss:-0.0006404975429177284 - actor/grad_norm:2.05078125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.231775760650635 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:79 - training/epoch:1 - critic/score/mean:0.4443359375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4443359375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.009245606139302254 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.009245606139302254 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.4609375 - response_length/max:128.0 - response_length/min:34.0 - response_length/clip_ratio:0.49609375 - response_length_non_aborted/mean:109.4609375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:34.0 - response_length_non_aborted/clip_ratio:0.49609375 - response/aborted_ratio:0.0 - prompt_length/mean:32.6796875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:8.224576968998008 - timing_s/old_log_prob:8.05561766900064 - timing_s/adv:0.18640233400219586 - timing_s/update_actor:25.63118657599989 - timing_s/update_weights:1.868054700003995 - timing_s/step:44.315183399005036 - timing_per_token_ms/adv:0.0012806580054014775 - timing_per_token_ms/update_actor:0.17609642310651788 - timing_per_token_ms/gen:0.07337607031080944 - perf/total_num_tokens:145552 - perf/time_per_step:44.315183399005036 - perf/throughput:3284.472472774826 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:80 - global_seqlen/min:147713.0 - global_seqlen/max:147713.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147713.0 - global_seqlen/balanced_max:147713.0 - global_seqlen/mean:147713.0 - actor/entropy:0.14050577580928802 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5746644735336304 - training/rollout_probs_diff_mean:0.004567266441881657 - training/rollout_probs_diff_std:0.015060347504913807 - training/rollout_actor_probs_pearson_corr:0.9966818690299988 - rollout_corr/training_ppl:1.161912202835083 - rollout_corr/training_log_ppl:0.14752575755119324 - rollout_corr/kl:0.0011934703215956688 - rollout_corr/k3_kl:0.0009227479458786547 - rollout_corr/rollout_ppl:1.1605098247528076 - rollout_corr/rollout_log_ppl:0.14636732637882233 - rollout_corr/log_ppl_diff:0.0011584204621613026 - rollout_corr/log_ppl_abs_diff:0.0033515188843011856 - rollout_corr/log_ppl_diff_max:0.027387067675590515 - rollout_corr/log_ppl_diff_min:-0.01638181507587433 - rollout_corr/ppl_ratio:1.001168966293335 - rollout_corr/chi2_token:0.0012952089309692383 - rollout_corr/chi2_seq:0.11503469944000244 - actor/pg_clipfrac:0.0024916750262491405 - actor/ppo_kl:-3.24493756428031e-06 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0021447438575705746 - actor/loss:0.002144742524251342 - actor/grad_norm:1.8828125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.14879035949707 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.462) - val-core/char_count/acc/mean@1:np.float64(0.462) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:80 - training/epoch:1 - critic/score/mean:0.3994140625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3994140625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005108699202537537 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005108699202537537 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:111.1416015625 - response_length/max:128.0 - response_length/min:41.0 - response_length/clip_ratio:0.5341796875 - response_length_non_aborted/mean:111.1416015625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:41.0 - response_length_non_aborted/clip_ratio:0.5341796875 - response/aborted_ratio:0.0 - prompt_length/mean:33.109375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.188668907998363 - timing_s/old_log_prob:8.454995386004157 - timing_s/adv:0.180486737001047 - timing_s/update_actor:26.132294811002794 - timing_s/update_weights:1.8922050129986019 - timing_s/step:45.04814207099844 - timing_s/testing:9.00672492099693 - timing_per_token_ms/adv:0.001221874425413112 - timing_per_token_ms/update_actor:0.17691262658671067 - timing_per_token_ms/gen:0.07195097846390323 - perf/total_num_tokens:147713 - perf/time_per_step:45.04814207099844 - perf/throughput:3279.003155495201 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 58%|█████▊ | 81/140 [1:05:00<46:36, 47.40s/it] (TransferQueueController pid=2438) 2026-11-01 18:21:17,057 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 28679, Total req/min: 5585.83, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.13 minutes; (TransferQueueController pid=2438) Per-operation statistics: SET_CUSTOM_META: req_count=10708, req/min=2085.60, avg_time=0.000009s, max_time=0.000211s, min_time=0.000003s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.36, avg_time=0.000114s, max_time=0.000121s, min_time=0.000108s; KV_RETRIEVE_META: req_count=10751, req/min=2093.98, avg_time=0.000266s, max_time=0.066496s, min_time=0.000063s; NOTIFY_DATA_UPDATE: req_count=7164, req/min=1395.34, avg_time=0.000083s, max_time=0.000745s, min_time=0.000031s; CLEAR_META: req_count=14, req/min=2.73, avg_time=0.001818s, max_time=0.003687s, min_time=0.000846s; KV_LIST: req_count=35, req/min=6.82, avg_time=0.000240s, max_time=0.001041s, min_time=0.000066s (TaskRunnerV1 pid=2351) Training Progress: 59%|█████▊ | 82/140 [1:05:45<45:09, 46.71s/it] (SimpleStorageUnit pid=2599) Per-operation statistics: GET_DATA: req_count=36, req/min=7.02, avg_time=0.000617s, max_time=0.001176s, min_time=0.000399s; PUT_DATA: req_count=914, req/min=178.33, avg_time=0.000616s, max_time=0.002191s, min_time=0.000316s; CLEAR_DATA: req_count=14, req/min=2.73, avg_time=0.001143s, max_time=0.001993s, min_time=0.000386s (SimpleStorageUnit pid=2599) 2026-11-01 18:22:01,566 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 964, Total req/min: 188.09, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.13 minutes; (SimpleStorageUnit pid=2596) 2026-11-01 18:22:01,565 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_72b4f821: [Performance] Total success requests: 959, Total req/min: 187.12, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2596) Time range: last 5.13 minutes; (TaskRunnerV1 pid=2351) Training Progress: 59%|█████▉ | 83/140 [1:06:30<43:45, 46.06s/it] (TaskRunnerV1 pid=2351) step:81 - global_seqlen/min:149125.0 - global_seqlen/max:149125.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:149125.0 - global_seqlen/balanced_max:149125.0 - global_seqlen/mean:149125.0 - actor/entropy:0.1345004290342331 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9996933937072754 - training/rollout_probs_diff_mean:0.004519747104495764 - training/rollout_probs_diff_std:0.01750933937728405 - training/rollout_actor_probs_pearson_corr:0.9956327080726624 - rollout_corr/training_ppl:1.154576301574707 - rollout_corr/training_log_ppl:0.14139977097511292 - rollout_corr/kl:0.0010112863965332508 - rollout_corr/k3_kl:0.0014156295219436288 - rollout_corr/rollout_ppl:1.1533819437026978 - rollout_corr/rollout_log_ppl:0.140421062707901 - rollout_corr/log_ppl_diff:0.0009787131566554308 - rollout_corr/log_ppl_abs_diff:0.0036061604041606188 - rollout_corr/log_ppl_diff_max:0.07428547739982605 - rollout_corr/log_ppl_diff_min:-0.0240696519613266 - rollout_corr/ppl_ratio:1.000996470451355 - rollout_corr/chi2_token:0.006239771842956543 - rollout_corr/chi2_seq:1.0547587871551514 - actor/pg_clipfrac:0.003242137422300099 - actor/ppo_kl:0.00026920588327546585 - actor/pg_clipfrac_lower:7.593706565482232e-06 - actor/pg_loss:0.00028002020553685725 - actor/loss:0.0002800207585096359 - actor/grad_norm:2.0185546875 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.17374563217163 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:81 - training/epoch:1 - critic/score/mean:0.3974609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.3974609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008487333543598652 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008487333543598652 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:113.0673828125 - response_length/max:128.0 - response_length/min:35.0 - response_length/clip_ratio:0.5361328125 - response_length_non_aborted/mean:113.0673828125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:35.0 - response_length_non_aborted/clip_ratio:0.5361328125 - response/aborted_ratio:0.0 - prompt_length/mean:32.5625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0234375 - timing_s/gen:8.200298765994376 - timing_s/old_log_prob:8.248186719996738 - timing_s/adv:0.18272310300380923 - timing_s/update_actor:26.42731467900012 - timing_s/update_weights:1.8743389880037284 - timing_s/step:45.10054640300223 - timing_per_token_ms/adv:0.0012253016127665329 - timing_per_token_ms/update_actor:0.17721585702598572 - timing_per_token_ms/gen:0.07082594524139864 - perf/total_num_tokens:149125 - perf/time_per_step:45.10054640300223 - perf/throughput:3306.500960486659 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:82 - global_seqlen/min:147528.0 - global_seqlen/max:147528.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147528.0 - global_seqlen/balanced_max:147528.0 - global_seqlen/mean:147528.0 - actor/entropy:0.13398785889148712 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9031493067741394 - training/rollout_probs_diff_mean:0.00447668693959713 - training/rollout_probs_diff_std:0.015968281775712967 - training/rollout_actor_probs_pearson_corr:0.9963271021842957 - rollout_corr/training_ppl:1.1569128036499023 - rollout_corr/training_log_ppl:0.1430780291557312 - rollout_corr/kl:0.000990216969512403 - rollout_corr/k3_kl:0.0010728799970820546 - rollout_corr/rollout_ppl:1.1557068824768066 - rollout_corr/rollout_log_ppl:0.1420551836490631 - rollout_corr/log_ppl_diff:0.0010228543542325497 - rollout_corr/log_ppl_abs_diff:0.0035492200404405594 - rollout_corr/log_ppl_diff_max:0.08067156374454498 - rollout_corr/log_ppl_diff_min:-0.015496328473091125 - rollout_corr/ppl_ratio:1.0010396242141724 - rollout_corr/chi2_token:0.0021284818649291992 - rollout_corr/chi2_seq:0.2249748706817627 - actor/pg_clipfrac:0.0029361119668465108 - actor/ppo_kl:8.200249014104582e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0023306633229367435 - actor/loss:0.0023306605871766806 - actor/grad_norm:1.9892578125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.22257900238037 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:82 - training/epoch:1 - critic/score/mean:0.435546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.435546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.010429986752569675 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.010429986752569675 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:111.1171875 - response_length/max:128.0 - response_length/min:26.0 - response_length/clip_ratio:0.53125 - response_length_non_aborted/mean:111.1171875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:26.0 - response_length_non_aborted/clip_ratio:0.53125 - response/aborted_ratio:0.0 - prompt_length/mean:32.953125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.197558783002023 - timing_s/old_log_prob:8.24881166999694 - timing_s/adv:0.19068933700327761 - timing_s/update_actor:26.270601210002496 - timing_s/update_weights:1.8967157959996257 - timing_s/step:44.97563723400526 - timing_per_token_ms/adv:0.0012925636964052764 - timing_per_token_ms/update_actor:0.17807196742315015 - timing_per_token_ms/gen:0.07204491653485572 - perf/total_num_tokens:147528 - perf/time_per_step:44.97563723400526 - perf/throughput:3280.175870158806 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:83 - global_seqlen/min:146384.0 - global_seqlen/max:146384.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146384.0 - global_seqlen/balanced_max:146384.0 - global_seqlen/mean:146384.0 - actor/entropy:0.13567879796028137 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7187100648880005 - training/rollout_probs_diff_mean:0.004652117844671011 - training/rollout_probs_diff_std:0.016155807301402092 - training/rollout_actor_probs_pearson_corr:0.9961803555488586 - rollout_corr/training_ppl:1.1551196575164795 - rollout_corr/training_log_ppl:0.14205090701580048 - rollout_corr/kl:0.001325677614659071 - rollout_corr/k3_kl:0.0011638968717306852 - rollout_corr/rollout_ppl:1.1535356044769287 - rollout_corr/rollout_log_ppl:0.14070294797420502 - rollout_corr/log_ppl_diff:0.001347956946119666 - rollout_corr/log_ppl_abs_diff:0.0037127581890672445 - rollout_corr/log_ppl_diff_max:0.08269071578979492 - rollout_corr/log_ppl_diff_min:-0.015422552824020386 - rollout_corr/ppl_ratio:1.001366376876831 - rollout_corr/chi2_token:0.0016654729843139648 - rollout_corr/chi2_seq:0.1947183609008789 - actor/pg_clipfrac:0.00322454737033695 - actor/ppo_kl:-0.00012090698601241456 - actor/pg_clipfrac_lower:4.222999965956357e-05 - actor/pg_loss:0.0016622587936581112 - actor/loss:0.0016622582916170359 - actor/grad_norm:1.8134765625 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.21456289291382 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:83 - training/epoch:1 - critic/score/mean:0.466796875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.466796875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0001318345166509971 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.0001318345166509971 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:110.828125 - response_length/max:128.0 - response_length/min:36.0 - response_length/clip_ratio:0.4443359375 - response_length_non_aborted/mean:110.828125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:36.0 - response_length_non_aborted/clip_ratio:0.4443359375 - response/aborted_ratio:0.0 - prompt_length/mean:32.125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.210479510998994 - timing_s/old_log_prob:8.129504436001298 - timing_s/adv:0.18416485199850285 - timing_s/update_actor:25.798206398998445 - timing_s/update_weights:1.8824761249998119 - timing_s/step:44.372249517000455 - timing_per_token_ms/adv:0.001258094135960917 - timing_per_token_ms/update_actor:0.17623651764536047 - timing_per_token_ms/gen:0.07234667551634529 - perf/total_num_tokens:146384 - perf/time_per_step:44.372249517000455 - perf/throughput:3298.998847104101 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 60%|██████ | 84/140 [1:07:15<42:46, 45.83s/it] (SimpleStorageUnit pid=2583) Per-operation statistics: GET_DATA: req_count=36, req/min=7.02, avg_time=0.000654s, max_time=0.001262s, min_time=0.000376s; PUT_DATA: req_count=915, req/min=178.53, avg_time=0.000561s, max_time=0.003399s, min_time=0.000310s; CLEAR_DATA: req_count=14, req/min=2.73, avg_time=0.001810s, max_time=0.004383s, min_time=0.000405s [repeated 7x across cluster] (SimpleStorageUnit pid=2583) 2026-11-01 18:22:01,566 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 965, Total req/min: 188.29, Total avg process time: 0.0006s; [repeated 6x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.13 minutes; [repeated 6x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 61%|██████ | 85/140 [1:08:09<44:12, 48.24s/it] (TaskRunnerV1 pid=2351) Training Progress: 61%|██████▏ | 86/140 [1:08:53<42:14, 46.94s/it] (TaskRunnerV1 pid=2351) step:84 - global_seqlen/min:147549.0 - global_seqlen/max:147549.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147549.0 - global_seqlen/balanced_max:147549.0 - global_seqlen/mean:147549.0 - actor/entropy:0.1349061131477356 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6819490194320679 - training/rollout_probs_diff_mean:0.004546887706965208 - training/rollout_probs_diff_std:0.015520554035902023 - training/rollout_actor_probs_pearson_corr:0.9964371919631958 - rollout_corr/training_ppl:1.1564387083053589 - rollout_corr/training_log_ppl:0.14322607219219208 - rollout_corr/kl:0.0011563902953639627 - rollout_corr/k3_kl:0.0009350820328108966 - rollout_corr/rollout_ppl:1.155091404914856 - rollout_corr/rollout_log_ppl:0.14210441708564758 - rollout_corr/log_ppl_diff:0.001121637993492186 - rollout_corr/log_ppl_abs_diff:0.0035649826750159264 - rollout_corr/log_ppl_diff_max:0.02689366042613983 - rollout_corr/log_ppl_diff_min:-0.034948572516441345 - rollout_corr/ppl_ratio:1.0011337995529175 - rollout_corr/chi2_token:0.0014882087707519531 - rollout_corr/chi2_seq:0.2069634199142456 - actor/pg_clipfrac:0.003198991869188224 - actor/ppo_kl:5.4853065194038215e-05 - actor/pg_clipfrac_lower:7.75049617611027e-06 - actor/pg_loss:-0.002257118874695152 - actor/loss:-0.0022571198642253876 - actor/grad_norm:1.86328125 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.221920013427734 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:84 - training/epoch:1 - critic/score/mean:0.4443359375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4443359375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.01025878544896841 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.01025878544896841 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:111.7001953125 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.533203125 - response_length_non_aborted/mean:111.7001953125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.533203125 - response/aborted_ratio:0.0 - prompt_length/mean:32.390625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.1910813420036 - timing_s/old_log_prob:8.455850610997004 - timing_s/adv:0.1819381390014314 - timing_s/update_actor:26.299039347999496 - timing_s/update_weights:1.8760460590056027 - timing_s/step:45.17492636300449 - timing_per_token_ms/adv:0.001233069278690004 - timing_per_token_ms/update_actor:0.17823936013120723 - timing_per_token_ms/gen:0.07161225502490448 - perf/total_num_tokens:147549 - perf/time_per_step:45.17492636300449 - perf/throughput:3266.1702382061576 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:85 - global_seqlen/min:147385.0 - global_seqlen/max:147385.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147385.0 - global_seqlen/balanced_max:147385.0 - global_seqlen/mean:147385.0 - actor/entropy:0.13502264022827148 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6937320232391357 - training/rollout_probs_diff_mean:0.004483283497393131 - training/rollout_probs_diff_std:0.015393047593533993 - training/rollout_actor_probs_pearson_corr:0.9964917898178101 - rollout_corr/training_ppl:1.15354323387146 - rollout_corr/training_log_ppl:0.14040647447109222 - rollout_corr/kl:0.0007764947949908674 - rollout_corr/k3_kl:0.0008882799302227795 - rollout_corr/rollout_ppl:1.1526305675506592 - rollout_corr/rollout_log_ppl:0.13964511454105377 - rollout_corr/log_ppl_diff:0.0007613656343892217 - rollout_corr/log_ppl_abs_diff:0.003235995303839445 - rollout_corr/log_ppl_diff_max:0.018998295068740845 - rollout_corr/log_ppl_diff_min:-0.02205720543861389 - rollout_corr/ppl_ratio:1.000771164894104 - rollout_corr/chi2_token:0.001998424530029297 - rollout_corr/chi2_seq:0.2909395694732666 - actor/pg_clipfrac:0.0033048681846897425 - actor/ppo_kl:7.558892704651043e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0022573063615709543 - actor/loss:0.002257306594401598 - actor/grad_norm:1.8701171875 - actor/perf/max_memory_allocated_gb:21.67309045791626 - actor/perf/max_memory_reserved_gb:28.345703125 - actor/perf/cpu_memory_used_gb:42.185848236083984 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.47) - val-core/char_count/acc/mean@1:np.float64(0.47) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:85 - training/epoch:1 - critic/score/mean:0.447265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.447265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004248285666108131 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004248285666108131 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:111.4775390625 - response_length/max:128.0 - response_length/min:29.0 - response_length/clip_ratio:0.501953125 - response_length_non_aborted/mean:111.4775390625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:29.0 - response_length_non_aborted/clip_ratio:0.501953125 - response/aborted_ratio:0.0 - prompt_length/mean:32.453125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.182184149998648 - timing_s/old_log_prob:8.285636861997773 - timing_s/adv:0.38123713200184284 - timing_s/update_actor:25.915942738000012 - timing_s/update_weights:1.8799588840047363 - timing_s/step:44.816221132001374 - timing_s/testing:8.68322412600537 - timing_per_token_ms/adv:0.0025866752519038087 - timing_per_token_ms/update_actor:0.17583840104488255 - timing_per_token_ms/gen:0.07167734663126372 - perf/total_num_tokens:147385 - perf/time_per_step:44.816221132001374 - perf/throughput:3288.652998339447 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:86 - global_seqlen/min:143738.0 - global_seqlen/max:143738.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:143738.0 - global_seqlen/balanced_max:143738.0 - global_seqlen/mean:143738.0 - actor/entropy:0.13698923587799072 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.681950032711029 - training/rollout_probs_diff_mean:0.004523633047938347 - training/rollout_probs_diff_std:0.01498854998499155 - training/rollout_actor_probs_pearson_corr:0.9966463446617126 - rollout_corr/training_ppl:1.1586859226226807 - rollout_corr/training_log_ppl:0.14499936997890472 - rollout_corr/kl:0.0009612208232283592 - rollout_corr/k3_kl:0.0009449014323763549 - rollout_corr/rollout_ppl:1.1575002670288086 - rollout_corr/rollout_log_ppl:0.14400921761989594 - rollout_corr/log_ppl_diff:0.0009901468874886632 - rollout_corr/log_ppl_abs_diff:0.0035862415097653866 - rollout_corr/log_ppl_diff_max:0.037119485437870026 - rollout_corr/log_ppl_diff_min:-0.022239625453948975 - rollout_corr/ppl_ratio:1.0010031461715698 - rollout_corr/chi2_token:0.0017421245574951172 - rollout_corr/chi2_seq:0.26153862476348877 - actor/pg_clipfrac:0.0028430055196319395 - actor/ppo_kl:-0.00014942331002506154 - actor/pg_clipfrac_lower:2.3046859496389516e-05 - actor/pg_loss:0.0014040126116015017 - actor/loss:0.0014040125533938408 - actor/grad_norm:1.66015625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.29200077056885 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:86 - training/epoch:1 - critic/score/mean:0.44140625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.44140625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008149470202624798 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008149470202624798 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:108.048828125 - response_length/max:128.0 - response_length/min:28.0 - response_length/clip_ratio:0.4609375 - response_length_non_aborted/mean:108.048828125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:28.0 - response_length_non_aborted/clip_ratio:0.4609375 - response/aborted_ratio:0.0 - prompt_length/mean:32.3203125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.199729311003466 - timing_s/old_log_prob:8.080666770998505 - timing_s/adv:0.1939978020018316 - timing_s/update_actor:25.265594508004142 - timing_s/update_weights:1.8851472969981842 - timing_s/step:43.79697795100219 - timing_per_token_ms/adv:0.0013496625944554091 - timing_per_token_ms/update_actor:0.1757753308659098 - timing_per_token_ms/gen:0.07411045815335465 - perf/total_num_tokens:143738 - perf/time_per_step:43.79697795100219 - perf/throughput:3281.9159386021265 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 62%|██████▏ | 87/140 [1:09:37<40:46, 46.17s/it] (TaskRunnerV1 pid=2351) Training Progress: 63%|██████▎ | 88/140 [1:10:22<39:39, 45.77s/it] (TransferQueueController pid=2438) 2026-11-01 18:26:38,681 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 32276, Total req/min: 6021.19, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.36 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=12035, req/min=2245.17, avg_time=0.000246s, max_time=0.018971s, min_time=0.000064s; CLEAR_META: req_count=16, req/min=2.98, avg_time=0.001664s, max_time=0.003913s, min_time=0.000863s; SET_CUSTOM_META: req_count=11989, req/min=2236.59, avg_time=0.000009s, max_time=0.000511s, min_time=0.000003s; KV_LIST: req_count=40, req/min=7.46, avg_time=0.000228s, max_time=0.000979s, min_time=0.000036s; NOTIFY_DATA_UPDATE: req_count=8189, req/min=1527.69, avg_time=0.000081s, max_time=0.000652s, min_time=0.000031s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.31, avg_time=0.000129s, max_time=0.000213s, min_time=0.000108s (TaskRunnerV1 pid=2351) Training Progress: 64%|██████▎ | 89/140 [1:11:07<38:39, 45.49s/it] (TaskRunnerV1 pid=2351) step:87 - global_seqlen/min:145024.0 - global_seqlen/max:145024.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:145024.0 - global_seqlen/balanced_max:145024.0 - global_seqlen/mean:145024.0 - actor/entropy:0.13966967165470123 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6084787845611572 - training/rollout_probs_diff_mean:0.004701063968241215 - training/rollout_probs_diff_std:0.01568014919757843 - training/rollout_actor_probs_pearson_corr:0.996382474899292 - rollout_corr/training_ppl:1.1598960161209106 - rollout_corr/training_log_ppl:0.14601971209049225 - rollout_corr/kl:0.0009824775625020266 - rollout_corr/k3_kl:0.0009524974157102406 - rollout_corr/rollout_ppl:1.158681869506836 - rollout_corr/rollout_log_ppl:0.14501766860485077 - rollout_corr/log_ppl_diff:0.0010020503541454673 - rollout_corr/log_ppl_abs_diff:0.003449623007327318 - rollout_corr/log_ppl_diff_max:0.022847980260849 - rollout_corr/log_ppl_diff_min:-0.02086000144481659 - rollout_corr/ppl_ratio:1.0010128021240234 - rollout_corr/chi2_token:0.0018721818923950195 - rollout_corr/chi2_seq:0.18084442615509033 - actor/pg_clipfrac:0.003521291859215126 - actor/ppo_kl:-0.00017713714328237984 - actor/pg_clipfrac_lower:7.66494971079131e-06 - actor/pg_loss:0.0018687138363020495 - actor/loss:0.0018687134142965078 - actor/grad_norm:1.931640625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.20842361450195 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:87 - training/epoch:1 - critic/score/mean:0.4794921875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4794921875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.002009414369240403 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.002009414369240403 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.0390625 - response_length/max:128.0 - response_length/min:33.0 - response_length/clip_ratio:0.4755859375 - response_length_non_aborted/mean:109.0390625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:33.0 - response_length_non_aborted/clip_ratio:0.4755859375 - response/aborted_ratio:0.0 - prompt_length/mean:32.5859375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.19297810200078 - timing_s/old_log_prob:8.2298482039987 - timing_s/adv:0.187699424001039 - timing_s/update_actor:25.554022541997256 - timing_s/update_weights:1.888032493996434 - timing_s/step:44.23263968099491 - timing_per_token_ms/adv:0.0012942645631139605 - timing_per_token_ms/update_actor:0.17620547317683458 - timing_per_token_ms/gen:0.07337696229491276 - perf/total_num_tokens:145024 - perf/time_per_step:44.23263968099491 - perf/throughput:3278.664828640361 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:88 - global_seqlen/min:147174.0 - global_seqlen/max:147174.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147174.0 - global_seqlen/balanced_max:147174.0 - global_seqlen/mean:147174.0 - actor/entropy:0.13741721212863922 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9729525446891785 - training/rollout_probs_diff_mean:0.004664925392717123 - training/rollout_probs_diff_std:0.016775570809841156 - training/rollout_actor_probs_pearson_corr:0.9959847927093506 - rollout_corr/training_ppl:1.160262942314148 - rollout_corr/training_log_ppl:0.14596673846244812 - rollout_corr/kl:0.0010007850360125303 - rollout_corr/k3_kl:0.0010862959316000342 - rollout_corr/rollout_ppl:1.1590083837509155 - rollout_corr/rollout_log_ppl:0.14492882788181305 - rollout_corr/log_ppl_diff:0.001037922571413219 - rollout_corr/log_ppl_abs_diff:0.0035757471341639757 - rollout_corr/log_ppl_diff_max:0.04469309002161026 - rollout_corr/log_ppl_diff_min:-0.019752025604248047 - rollout_corr/ppl_ratio:1.0010521411895752 - rollout_corr/chi2_token:0.0020563602447509766 - rollout_corr/chi2_seq:0.24665403366088867 - actor/pg_clipfrac:0.0032761781573450817 - actor/ppo_kl:-0.00013795834651849267 - actor/pg_clipfrac_lower:4.910768924067573e-05 - actor/pg_loss:0.0006509641389129683 - actor/loss:0.0006509635131806135 - actor/grad_norm:1.9130859375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.19256114959717 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:88 - training/epoch:1 - critic/score/mean:0.4462890625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4462890625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.004357797093689442 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.004357797093689442 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:110.888671875 - response_length/max:128.0 - response_length/min:41.0 - response_length/clip_ratio:0.4970703125 - response_length_non_aborted/mean:110.888671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:41.0 - response_length_non_aborted/clip_ratio:0.4970703125 - response/aborted_ratio:0.0 - prompt_length/mean:32.8359375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.193396736001887 - timing_s/old_log_prob:8.312548670000979 - timing_s/adv:0.18867755600513192 - timing_s/update_actor:25.94963282700337 - timing_s/update_weights:1.87969716200314 - timing_s/step:44.69331497500389 - timing_per_token_ms/adv:0.0012820033158379327 - timing_per_token_ms/update_actor:0.17631940986181913 - timing_per_token_ms/gen:0.07215673039191446 - perf/total_num_tokens:147174 - perf/time_per_step:44.69331497500389 - perf/throughput:3292.975696305177 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:89 - global_seqlen/min:146978.0 - global_seqlen/max:146978.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146978.0 - global_seqlen/balanced_max:146978.0 - global_seqlen/mean:146978.0 - actor/entropy:0.134382426738739 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9814577698707581 - training/rollout_probs_diff_mean:0.004522983450442553 - training/rollout_probs_diff_std:0.01578543335199356 - training/rollout_actor_probs_pearson_corr:0.9962407946586609 - rollout_corr/training_ppl:1.1549410820007324 - rollout_corr/training_log_ppl:0.14170798659324646 - rollout_corr/kl:0.0010037674801424146 - rollout_corr/k3_kl:0.001036785775795579 - rollout_corr/rollout_ppl:1.1537009477615356 - rollout_corr/rollout_log_ppl:0.14068369567394257 - rollout_corr/log_ppl_diff:0.0010242989519611 - rollout_corr/log_ppl_abs_diff:0.0034784015733748674 - rollout_corr/log_ppl_diff_max:0.06518261134624481 - rollout_corr/log_ppl_diff_min:-0.016840457916259766 - rollout_corr/ppl_ratio:1.0010390281677246 - rollout_corr/chi2_token:0.0020323991775512695 - rollout_corr/chi2_seq:0.25556719303131104 - actor/pg_clipfrac:0.003119728178717196 - actor/ppo_kl:-9.733010604880577e-06 - actor/pg_clipfrac_lower:1.5417269120613735e-05 - actor/pg_loss:0.0022497111713164486 - actor/loss:0.0022497111931443214 - actor/grad_norm:1.8125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.292826652526855 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:89 - training/epoch:1 - critic/score/mean:0.443359375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.443359375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00912626925855875 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.00912626925855875 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:110.955078125 - response_length/max:128.0 - response_length/min:29.0 - response_length/clip_ratio:0.51953125 - response_length_non_aborted/mean:110.955078125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:29.0 - response_length_non_aborted/clip_ratio:0.51953125 - response/aborted_ratio:0.0 - prompt_length/mean:32.578125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.185445732000517 - timing_s/old_log_prob:8.089423151999654 - timing_s/adv:0.38187333699897863 - timing_s/update_actor:25.980556230999355 - timing_s/update_weights:1.88673622499482 - timing_s/step:44.70304063599906 - timing_per_token_ms/adv:0.0025981666439805866 - timing_per_token_ms/update_actor:0.17676493237763036 - timing_per_token_ms/gen:0.0720435646816571 - perf/total_num_tokens:146978 - perf/time_per_step:44.70304063599906 - perf/throughput:3287.8747823171475 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(SimpleStorageUnit pid=2596) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=2.96, avg_time=0.001376s, max_time=0.003721s, min_time=0.000400s; PUT_DATA: req_count=1019, req/min=188.68, avg_time=0.000606s, max_time=0.001789s, min_time=0.000315s; GET_DATA: req_count=36, req/min=6.67, avg_time=0.000584s, max_time=0.001271s, min_time=0.000374s (SimpleStorageUnit pid=2596) 2026-11-01 18:27:25,603 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_72b4f821: [Performance] Total success requests: 1071, Total req/min: 198.31, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2596) Time range: last 5.40 minutes; (SimpleStorageUnit pid=2598) 2026-11-01 18:27:25,604 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_9f74a67b: [Performance] Total success requests: 1092, Total req/min: 202.20, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2598) Time range: last 5.40 minutes; (TaskRunnerV1 pid=2351) Training Progress: 64%|██████▍ | 90/140 [1:12:00<39:49, 47.80s/it] (SimpleStorageUnit pid=2597) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=2.95, avg_time=0.001341s, max_time=0.003904s, min_time=0.000386s; PUT_DATA: req_count=1056, req/min=194.91, avg_time=0.000613s, max_time=0.001448s, min_time=0.000308s; GET_DATA: req_count=36, req/min=6.64, avg_time=0.000614s, max_time=0.001248s, min_time=0.000405s [repeated 7x across cluster] (SimpleStorageUnit pid=2597) 2026-11-01 18:27:26,648 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_a2b08c31: [Performance] Total success requests: 1108, Total req/min: 204.50, Total avg process time: 0.0006s; [repeated 6x across cluster] (SimpleStorageUnit pid=2597) Time range: last 5.42 minutes; [repeated 6x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 65%|██████▌ | 91/140 [1:12:44<38:07, 46.69s/it] (TaskRunnerV1 pid=2351) Training Progress: 66%|██████▌ | 92/140 [1:13:29<36:55, 46.16s/it] (TaskRunnerV1 pid=2351) step:90 - global_seqlen/min:144872.0 - global_seqlen/max:144872.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:144872.0 - global_seqlen/balanced_max:144872.0 - global_seqlen/mean:144872.0 - actor/entropy:0.13324742019176483 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8065879344940186 - training/rollout_probs_diff_mean:0.004410066641867161 - training/rollout_probs_diff_std:0.014991496689617634 - training/rollout_actor_probs_pearson_corr:0.9966800808906555 - rollout_corr/training_ppl:1.151982307434082 - rollout_corr/training_log_ppl:0.13931357860565186 - rollout_corr/kl:0.000909050868358463 - rollout_corr/k3_kl:0.0008694558637216687 - rollout_corr/rollout_ppl:1.1508759260177612 - rollout_corr/rollout_log_ppl:0.1383909434080124 - rollout_corr/log_ppl_diff:0.0009226232650689781 - rollout_corr/log_ppl_abs_diff:0.0032868378330022097 - rollout_corr/log_ppl_diff_max:0.01875026524066925 - rollout_corr/log_ppl_diff_min:-0.01468716561794281 - rollout_corr/ppl_ratio:1.0009320974349976 - rollout_corr/chi2_token:0.0018311738967895508 - rollout_corr/chi2_seq:0.22317278385162354 - actor/pg_clipfrac:0.0029655817076369812 - actor/ppo_kl:0.0001684493950430503 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.001240269048139453 - actor/loss:0.0012402695138007402 - actor/grad_norm:1.7236328125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.2478461265564 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.479) - val-core/char_count/acc/mean@1:np.float64(0.479) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:90 - training/epoch:1 - critic/score/mean:0.4833984375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4833984375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007657209411263466 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007657209411263466 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.3671875 - response_length/max:128.0 - response_length/min:35.0 - response_length/clip_ratio:0.46875 - response_length_non_aborted/mean:109.3671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:35.0 - response_length_non_aborted/clip_ratio:0.46875 - response/aborted_ratio:0.0 - prompt_length/mean:32.109375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.2034469519931 - timing_s/old_log_prob:8.082317566004349 - timing_s/adv:0.38273792699328624 - timing_s/update_actor:25.544376294994436 - timing_s/update_weights:1.8970180570031516 - timing_s/step:44.28206735000276 - timing_s/testing:8.786925152002368 - timing_per_token_ms/adv:0.0026419040738947914 - timing_per_token_ms/update_actor:0.1763237637017121 - timing_per_token_ms/gen:0.07325029423524092 - perf/total_num_tokens:144872 - perf/time_per_step:44.28206735000276 - perf/throughput:3271.5726403408526 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:91 - global_seqlen/min:142683.0 - global_seqlen/max:142683.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142683.0 - global_seqlen/balanced_max:142683.0 - global_seqlen/mean:142683.0 - actor/entropy:0.1404053419828415 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9988197684288025 - training/rollout_probs_diff_mean:0.004744414240121841 - training/rollout_probs_diff_std:0.01601726934313774 - training/rollout_actor_probs_pearson_corr:0.9962725043296814 - rollout_corr/training_ppl:1.1631414890289307 - rollout_corr/training_log_ppl:0.1488288938999176 - rollout_corr/kl:0.0010663463035598397 - rollout_corr/k3_kl:0.0010845116339623928 - rollout_corr/rollout_ppl:1.1618340015411377 - rollout_corr/rollout_log_ppl:0.14773976802825928 - rollout_corr/log_ppl_diff:0.0010891146957874298 - rollout_corr/log_ppl_abs_diff:0.0036989725194871426 - rollout_corr/log_ppl_diff_max:0.1275303214788437 - rollout_corr/log_ppl_diff_min:-0.02739614248275757 - rollout_corr/ppl_ratio:1.001110315322876 - rollout_corr/chi2_token:0.0020589828491210938 - rollout_corr/chi2_seq:0.23322629928588867 - actor/pg_clipfrac:0.003732511897396762 - actor/ppo_kl:0.00010660543133174846 - actor/pg_clipfrac_lower:7.57025190978311e-06 - actor/pg_loss:0.0011738089815480635 - actor/loss:0.0011738084722310305 - actor/grad_norm:1.9375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.225032329559326 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:91 - training/epoch:1 - critic/score/mean:0.4853515625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4853515625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.004175033885985613 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.004175033885985613 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.1826171875 - response_length/max:128.0 - response_length/min:34.0 - response_length/clip_ratio:0.439453125 - response_length_non_aborted/mean:107.1826171875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:34.0 - response_length_non_aborted/clip_ratio:0.439453125 - response/aborted_ratio:0.0 - prompt_length/mean:32.15625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.209299556001497 - timing_s/old_log_prob:8.193565753994335 - timing_s/adv:0.18478442000196083 - timing_s/update_actor:25.26743514899863 - timing_s/update_weights:1.8922327700056485 - timing_s/step:43.91442330599966 - timing_per_token_ms/adv:0.0012950696298925648 - timing_per_token_ms/update_actor:0.17708791621285389 - timing_per_token_ms/gen:0.0747965883650084 - perf/total_num_tokens:142683 - perf/time_per_step:43.91442330599966 - perf/throughput:3249.1147385853615 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:92 - global_seqlen/min:147331.0 - global_seqlen/max:147331.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147331.0 - global_seqlen/balanced_max:147331.0 - global_seqlen/mean:147331.0 - actor/entropy:0.1333010047674179 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8807384371757507 - training/rollout_probs_diff_mean:0.004397499840706587 - training/rollout_probs_diff_std:0.015505768358707428 - training/rollout_actor_probs_pearson_corr:0.9964367747306824 - rollout_corr/training_ppl:1.1525737047195435 - rollout_corr/training_log_ppl:0.13957026600837708 - rollout_corr/kl:0.000706161605194211 - rollout_corr/k3_kl:0.0009216171456500888 - rollout_corr/rollout_ppl:1.151719093322754 - rollout_corr/rollout_log_ppl:0.1388741433620453 - rollout_corr/log_ppl_diff:0.0006961229373700917 - rollout_corr/log_ppl_abs_diff:0.003388487733900547 - rollout_corr/log_ppl_diff_max:0.02646350860595703 - rollout_corr/log_ppl_diff_min:-0.037140846252441406 - rollout_corr/ppl_ratio:1.0007070302963257 - rollout_corr/chi2_token:0.002438783645629883 - rollout_corr/chi2_seq:0.31738710403442383 - actor/pg_clipfrac:0.0030610834898349517 - actor/ppo_kl:0.00026271224656208386 - actor/pg_clipfrac_lower:7.75049617611027e-06 - actor/pg_loss:-0.0005247769622656051 - actor/loss:-0.0005247759399935603 - actor/grad_norm:1.78125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.2738561630249 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:92 - training/epoch:1 - critic/score/mean:0.41015625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.41015625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.012801315635442734 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.012801315635442734 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:110.5732421875 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.486328125 - response_length_non_aborted/mean:110.5732421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.486328125 - response/aborted_ratio:0.0 - prompt_length/mean:33.3046875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.190830362997076 - timing_s/old_log_prob:8.35789385499811 - timing_s/adv:0.1836295829998562 - timing_s/update_actor:25.984957654000027 - timing_s/update_weights:1.9247631909966003 - timing_s/step:44.81011676399794 - timing_per_token_ms/adv:0.0012463743747063159 - timing_per_token_ms/update_actor:0.17637128407463484 - timing_per_token_ms/gen:0.07233990446622339 - perf/total_num_tokens:147331 - perf/time_per_step:44.81011676399794 - perf/throughput:3287.895918146123 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 66%|██████▋ | 93/140 [1:14:14<35:50, 45.75s/it] (TaskRunnerV1 pid=2351) Training Progress: 67%|██████▋ | 94/140 [1:14:59<34:48, 45.41s/it] (TaskRunnerV1 pid=2351) Training Progress: 68%|██████▊ | 95/140 [1:15:52<35:48, 47.75s/it] (TaskRunnerV1 pid=2351) step:93 - global_seqlen/min:146614.0 - global_seqlen/max:146614.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146614.0 - global_seqlen/balanced_max:146614.0 - global_seqlen/mean:146614.0 - actor/entropy:0.13169342279434204 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.879810094833374 - training/rollout_probs_diff_mean:0.004479913040995598 - training/rollout_probs_diff_std:0.01551759708672762 - training/rollout_actor_probs_pearson_corr:0.9963626861572266 - rollout_corr/training_ppl:1.1507515907287598 - rollout_corr/training_log_ppl:0.13831791281700134 - rollout_corr/kl:0.0010895747691392899 - rollout_corr/k3_kl:0.0009862045990303159 - rollout_corr/rollout_ppl:1.1494768857955933 - rollout_corr/rollout_log_ppl:0.13726189732551575 - rollout_corr/log_ppl_diff:0.001056034816429019 - rollout_corr/log_ppl_abs_diff:0.0037154422607272863 - rollout_corr/log_ppl_diff_max:0.051333993673324585 - rollout_corr/log_ppl_diff_min:-0.0195201113820076 - rollout_corr/ppl_ratio:1.001070499420166 - rollout_corr/chi2_token:0.0020750761032104492 - rollout_corr/chi2_seq:0.3318380117416382 - actor/pg_clipfrac:0.0037251241932002204 - actor/ppo_kl:4.38707278505035e-05 - actor/pg_clipfrac_lower:1.568567252737315e-05 - actor/pg_loss:0.0014918327169652912 - actor/loss:0.001491833885665983 - actor/grad_norm:1.865234375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.23505401611328 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:93 - training/epoch:1 - critic/score/mean:0.4580078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4580078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006521193776279688 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006521193776279688 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:110.177734375 - response_length/max:128.0 - response_length/min:40.0 - response_length/clip_ratio:0.49609375 - response_length_non_aborted/mean:110.177734375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:40.0 - response_length_non_aborted/clip_ratio:0.49609375 - response/aborted_ratio:0.0 - prompt_length/mean:33.0 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.197956747993885 - timing_s/old_log_prob:8.110837870000978 - timing_s/adv:0.38414173899946036 - timing_s/update_actor:25.88157684900216 - timing_s/update_weights:1.9232648039978812 - timing_s/step:44.6666134029947 - timing_per_token_ms/adv:0.002620089070617133 - timing_per_token_ms/update_actor:0.17652868654427378 - timing_per_token_ms/gen:0.07266274971188141 - perf/total_num_tokens:146614 - perf/time_per_step:44.6666134029947 - perf/throughput:3282.4068992472617 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:94 - global_seqlen/min:145419.0 - global_seqlen/max:145419.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:145419.0 - global_seqlen/balanced_max:145419.0 - global_seqlen/mean:145419.0 - actor/entropy:0.1354646235704422 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7918661236763 - training/rollout_probs_diff_mean:0.004496488254517317 - training/rollout_probs_diff_std:0.015335869044065475 - training/rollout_actor_probs_pearson_corr:0.9965606331825256 - rollout_corr/training_ppl:1.156039834022522 - rollout_corr/training_log_ppl:0.14270761609077454 - rollout_corr/kl:0.000936919532250613 - rollout_corr/k3_kl:0.0009819852421060205 - rollout_corr/rollout_ppl:1.1548689603805542 - rollout_corr/rollout_log_ppl:0.14173872768878937 - rollout_corr/log_ppl_diff:0.0009688903810456395 - rollout_corr/log_ppl_abs_diff:0.003555453848093748 - rollout_corr/log_ppl_diff_max:0.040175661444664 - rollout_corr/log_ppl_diff_min:-0.023630768060684204 - rollout_corr/ppl_ratio:1.0009828805923462 - rollout_corr/chi2_token:0.0020481348037719727 - rollout_corr/chi2_seq:0.25435566902160645 - actor/pg_clipfrac:0.003346893558045849 - actor/ppo_kl:0.0001324921478650746 - actor/pg_clipfrac_lower:2.896145876244797e-05 - actor/pg_loss:0.0014426251000259072 - actor/loss:0.001442624256014824 - actor/grad_norm:2.1533203125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.29410219192505 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:94 - training/epoch:1 - critic/score/mean:0.494140625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.494140625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0018714306643232703 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0018714306643232703 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.6982421875 - response_length/max:128.0 - response_length/min:36.0 - response_length/clip_ratio:0.458984375 - response_length_non_aborted/mean:109.6982421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:36.0 - response_length_non_aborted/clip_ratio:0.458984375 - response/aborted_ratio:0.0 - prompt_length/mean:32.3125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.202154961996712 - timing_s/old_log_prob:8.128572843997972 - timing_s/adv:0.3797186900046654 - timing_s/update_actor:25.692429301998345 - timing_s/update_weights:1.9197376599986455 - timing_s/step:44.491949590999866 - timing_per_token_ms/adv:0.0026112041067856704 - timing_per_token_ms/update_actor:0.17667862729078282 - timing_per_token_ms/gen:0.07301773296771784 - perf/total_num_tokens:145419 - perf/time_per_step:44.491949590999866 - perf/throughput:3268.4339827045105 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:95 - global_seqlen/min:145389.0 - global_seqlen/max:145389.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:145389.0 - global_seqlen/balanced_max:145389.0 - global_seqlen/mean:145389.0 - actor/entropy:0.13551880419254303 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7663131952285767 - training/rollout_probs_diff_mean:0.00457749143242836 - training/rollout_probs_diff_std:0.015322348102927208 - training/rollout_actor_probs_pearson_corr:0.9966035485267639 - rollout_corr/training_ppl:1.1547762155532837 - rollout_corr/training_log_ppl:0.14166311919689178 - rollout_corr/kl:0.0008977623074315488 - rollout_corr/k3_kl:0.0009583645733073354 - rollout_corr/rollout_ppl:1.1537164449691772 - rollout_corr/rollout_log_ppl:0.14077305793762207 - rollout_corr/log_ppl_diff:0.0008900678367353976 - rollout_corr/log_ppl_abs_diff:0.003535949159413576 - rollout_corr/log_ppl_diff_max:0.036909520626068115 - rollout_corr/log_ppl_diff_min:-0.034039855003356934 - rollout_corr/ppl_ratio:1.0009026527404785 - rollout_corr/chi2_token:0.001978635787963867 - rollout_corr/chi2_seq:0.2768123149871826 - actor/pg_clipfrac:0.003337451213155873 - actor/ppo_kl:0.00018366246301108427 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:5.881700781174004e-05 - actor/loss:5.8816978707909584e-05 - actor/grad_norm:2.1123046875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.241047382354736 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.492) - val-core/char_count/acc/mean@1:np.float64(0.492) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:95 - training/epoch:1 - critic/score/mean:0.4541015625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4541015625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0026883152313530445 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0026883152313530445 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.4267578125 - response_length/max:128.0 - response_length/min:32.0 - response_length/clip_ratio:0.4326171875 - response_length_non_aborted/mean:109.4267578125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:32.0 - response_length_non_aborted/clip_ratio:0.4326171875 - response/aborted_ratio:0.0 - prompt_length/mean:32.5546875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.191112749002059 - timing_s/old_log_prob:8.060496525999042 - timing_s/adv:0.1916446450049989 - timing_s/update_actor:25.88664362199779 - timing_s/update_weights:1.8901729730059742 - timing_s/step:44.40759382199758 - timing_s/testing:8.674765182004194 - timing_per_token_ms/adv:0.001318150926170473 - timing_per_token_ms/update_actor:0.17805090909214447 - timing_per_token_ms/gen:0.07310034313228614 - perf/total_num_tokens:145389 - perf/time_per_step:44.40759382199758 - perf/throughput:3273.967073802153 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TransferQueueController pid=2438) 2026-11-01 18:31:39,903 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 39277, Total req/min: 7823.54, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.02 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=15034, req/min=2994.61, avg_time=0.000244s, max_time=0.080780s, min_time=0.000062s; CLEAR_META: req_count=18, req/min=3.59, avg_time=0.001749s, max_time=0.002221s, min_time=0.000851s; SET_CUSTOM_META: req_count=14987, req/min=2985.24, avg_time=0.000009s, max_time=0.000260s, min_time=0.000003s; KV_LIST: req_count=45, req/min=8.96, avg_time=0.000279s, max_time=0.000982s, min_time=0.000050s; NOTIFY_DATA_UPDATE: req_count=9187, req/min=1829.95, avg_time=0.000081s, max_time=0.000807s, min_time=0.000031s; KV_RETRIEVE_KEYS: req_count=6, req/min=1.20, avg_time=0.000126s, max_time=0.000180s, min_time=0.000109s (TaskRunnerV1 pid=2351) Training Progress: 69%|██████▊ | 96/140 [1:16:36<34:16, 46.74s/it] (SimpleStorageUnit pid=2599) 2026-11-01 18:32:51,639 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1190, Total req/min: 219.60, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.42 minutes; (SimpleStorageUnit pid=2599) Per-operation statistics: PUT_DATA: req_count=1132, req/min=208.89, avg_time=0.000617s, max_time=0.001674s, min_time=0.000316s; CLEAR_DATA: req_count=18, req/min=3.32, avg_time=0.001454s, max_time=0.004318s, min_time=0.000397s; GET_DATA: req_count=40, req/min=7.38, avg_time=0.000624s, max_time=0.001222s, min_time=0.000392s (TaskRunnerV1 pid=2351) Training Progress: 69%|██████▉ | 97/140 [1:17:20<32:50, 45.84s/it] (TaskRunnerV1 pid=2351) Training Progress: 70%|███████ | 98/140 [1:18:04<31:44, 45.35s/it] (SimpleStorageUnit pid=2583) 2026-11-01 18:32:51,640 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1228, Total req/min: 226.08, Total avg process time: 0.0006s; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.43 minutes; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: PUT_DATA: req_count=1170, req/min=215.41, avg_time=0.000606s, max_time=0.001619s, min_time=0.000318s; CLEAR_DATA: req_count=18, req/min=3.31, avg_time=0.001192s, max_time=0.002821s, min_time=0.000402s; GET_DATA: req_count=40, req/min=7.36, avg_time=0.000548s, max_time=0.000932s, min_time=0.000408s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) step:96 - global_seqlen/min:145423.0 - global_seqlen/max:145423.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:145423.0 - global_seqlen/balanced_max:145423.0 - global_seqlen/mean:145423.0 - actor/entropy:0.1283261626958847 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9773541688919067 - training/rollout_probs_diff_mean:0.004366742912679911 - training/rollout_probs_diff_std:0.01601257175207138 - training/rollout_actor_probs_pearson_corr:0.99617999792099 - rollout_corr/training_ppl:1.15049147605896 - rollout_corr/training_log_ppl:0.13809919357299805 - rollout_corr/kl:0.0010491115972399712 - rollout_corr/k3_kl:0.001001403434202075 - rollout_corr/rollout_ppl:1.1492176055908203 - rollout_corr/rollout_log_ppl:0.13702119886875153 - rollout_corr/log_ppl_diff:0.001077983295544982 - rollout_corr/log_ppl_abs_diff:0.0033089325297623873 - rollout_corr/log_ppl_diff_max:0.07267992198467255 - rollout_corr/log_ppl_diff_min:-0.010861307382583618 - rollout_corr/ppl_ratio:1.001090407371521 - rollout_corr/chi2_token:0.0016084909439086914 - rollout_corr/chi2_seq:0.13541531562805176 - actor/pg_clipfrac:0.0026690450564880543 - actor/ppo_kl:-0.00025194624853005126 - actor/pg_clipfrac_lower:1.7377449694322422e-05 - actor/pg_loss:0.0007984808762557805 - actor/loss:0.000798481865786016 - actor/grad_norm:1.76171875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.28333234786987 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:96 - training/epoch:1 - critic/score/mean:0.45703125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.45703125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.002318174345418811 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.002318174345418811 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.1162109375 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.478515625 - response_length_non_aborted/mean:109.1162109375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.478515625 - response/aborted_ratio:0.0 - prompt_length/mean:32.8984375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.21822765700199 - timing_s/old_log_prob:8.059699511999497 - timing_s/adv:0.4090350059996126 - timing_s/update_actor:25.49309629700292 - timing_s/update_weights:1.9052005659978022 - timing_s/step:44.25070833199425 - timing_per_token_ms/adv:0.002812725676128347 - timing_per_token_ms/update_actor:0.17530305589214168 - timing_per_token_ms/gen:0.07355105971273092 - perf/total_num_tokens:145423 - perf/time_per_step:44.25070833199425 - perf/throughput:3286.3428740835757 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:97 - global_seqlen/min:141522.0 - global_seqlen/max:141522.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:141522.0 - global_seqlen/balanced_max:141522.0 - global_seqlen/mean:141522.0 - actor/entropy:0.13792356848716736 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.5851343870162964 - training/rollout_probs_diff_mean:0.004632826428860426 - training/rollout_probs_diff_std:0.015221793204545975 - training/rollout_actor_probs_pearson_corr:0.9965614676475525 - rollout_corr/training_ppl:1.1558274030685425 - rollout_corr/training_log_ppl:0.1428072303533554 - rollout_corr/kl:0.000757404719479382 - rollout_corr/k3_kl:0.0008646867354400456 - rollout_corr/rollout_ppl:1.1548802852630615 - rollout_corr/rollout_log_ppl:0.14200758934020996 - rollout_corr/log_ppl_diff:0.0007996357744559646 - rollout_corr/log_ppl_abs_diff:0.0034850738011300564 - rollout_corr/log_ppl_diff_max:0.02435372769832611 - rollout_corr/log_ppl_diff_min:-0.016814738512039185 - rollout_corr/ppl_ratio:1.000810146331787 - rollout_corr/chi2_token:0.0019713640213012695 - rollout_corr/chi2_seq:0.2502717971801758 - actor/pg_clipfrac:0.003090791278130685 - actor/ppo_kl:0.0002800138978879355 - actor/pg_clipfrac_lower:7.574380409399358e-06 - actor/pg_loss:0.0002481177216395736 - actor/loss:0.00024811876937747 - actor/grad_norm:1.6943359375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.295936584472656 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:97 - training/epoch:1 - critic/score/mean:0.509765625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.509765625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006009347271174192 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006009347271174192 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:106.259765625 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.3896484375 - response_length_non_aborted/mean:106.259765625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.3896484375 - response/aborted_ratio:0.0 - prompt_length/mean:31.9453125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.203688765999686 - timing_s/old_log_prob:7.901728142001957 - timing_s/adv:0.38663994499802357 - timing_s/update_actor:25.03483124000195 - timing_s/update_weights:1.8900034429971129 - timing_s/step:43.59697668800072 - timing_per_token_ms/adv:0.002732013008564206 - timing_per_token_ms/update_actor:0.17689709896695885 - timing_per_token_ms/gen:0.07539462150537346 - perf/total_num_tokens:141522 - perf/time_per_step:43.59697668800072 - perf/throughput:3246.142525267156 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:98 - global_seqlen/min:144002.0 - global_seqlen/max:144002.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:144002.0 - global_seqlen/balanced_max:144002.0 - global_seqlen/mean:144002.0 - actor/entropy:0.135745108127594 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7543227672576904 - training/rollout_probs_diff_mean:0.004574810154736042 - training/rollout_probs_diff_std:0.015908807516098022 - training/rollout_actor_probs_pearson_corr:0.9962677955627441 - rollout_corr/training_ppl:1.1547163724899292 - rollout_corr/training_log_ppl:0.1417195051908493 - rollout_corr/kl:0.0010866011725738645 - rollout_corr/k3_kl:0.0010051802964881063 - rollout_corr/rollout_ppl:1.1532983779907227 - rollout_corr/rollout_log_ppl:0.1405515968799591 - rollout_corr/log_ppl_diff:0.0011679090093821287 - rollout_corr/log_ppl_abs_diff:0.0037920561153441668 - rollout_corr/log_ppl_diff_max:0.025444969534873962 - rollout_corr/log_ppl_diff_min:-0.01683652400970459 - rollout_corr/ppl_ratio:1.0011814832687378 - rollout_corr/chi2_token:0.0017406940460205078 - rollout_corr/chi2_seq:0.27350616455078125 - actor/pg_clipfrac:0.0034543527629769715 - actor/ppo_kl:9.165771886425013e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0009475520928390324 - actor/loss:0.0009475531987845898 - actor/grad_norm:1.794921875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.185075759887695 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:98 - training/epoch:1 - critic/score/mean:0.462890625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.462890625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.01610579527914524 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.01610579527914524 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.603515625 - response_length/max:128.0 - response_length/min:41.0 - response_length/clip_ratio:0.4375 - response_length_non_aborted/mean:107.603515625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:41.0 - response_length_non_aborted/clip_ratio:0.4375 - response/aborted_ratio:0.0 - prompt_length/mean:33.0234375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.09375 - timing_s/gen:8.181800624006428 - timing_s/old_log_prob:8.008436272000836 - timing_s/adv:0.23983532100101002 - timing_s/update_actor:25.58762259800278 - timing_s/update_weights:1.9075264080020133 - timing_s/step:44.09451454599912 - timing_per_token_ms/adv:0.001665499930563534 - timing_per_token_ms/update_actor:0.17768935568952363 - timing_per_token_ms/gen:0.07425444815136613 - perf/total_num_tokens:144002 - perf/time_per_step:44.09451454599912 - perf/throughput:3265.7576907843722 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 71%|███████ | 99/140 [1:18:49<30:58, 45.33s/it] (TaskRunnerV1 pid=2351) Training Progress: 71%|███████▏ | 100/140 [1:19:42<31:42, 47.57s/it] (TaskRunnerV1 pid=2351) Training Progress: 72%|███████▏ | 101/140 [1:20:26<30:14, 46.53s/it] (TaskRunnerV1 pid=2351) step:99 - global_seqlen/min:147768.0 - global_seqlen/max:147768.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:147768.0 - global_seqlen/balanced_max:147768.0 - global_seqlen/mean:147768.0 - actor/entropy:0.13084885478019714 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.909974217414856 - training/rollout_probs_diff_mean:0.0042888750322163105 - training/rollout_probs_diff_std:0.014869138598442078 - training/rollout_actor_probs_pearson_corr:0.9965565204620361 - rollout_corr/training_ppl:1.148518681526184 - rollout_corr/training_log_ppl:0.13629701733589172 - rollout_corr/kl:0.000991690088994801 - rollout_corr/k3_kl:0.0008808596758171916 - rollout_corr/rollout_ppl:1.1473281383514404 - rollout_corr/rollout_log_ppl:0.135295107960701 - rollout_corr/log_ppl_diff:0.0010018977336585522 - rollout_corr/log_ppl_abs_diff:0.0033305310644209385 - rollout_corr/log_ppl_diff_max:0.024864092469215393 - rollout_corr/log_ppl_diff_min:-0.01929868757724762 - rollout_corr/ppl_ratio:1.0010128021240234 - rollout_corr/chi2_token:0.001580953598022461 - rollout_corr/chi2_seq:0.3242626190185547 - actor/pg_clipfrac:0.002868326574874421 - actor/ppo_kl:4.281001277680237e-05 - actor/pg_clipfrac_lower:7.762046758822786e-06 - actor/pg_loss:0.0020836709736613557 - actor/loss:0.0020836705807596445 - actor/grad_norm:1.7900390625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.21510028839111 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:99 - training/epoch:1 - critic/score/mean:0.44140625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.44140625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007340432610362768 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007340432610362768 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:111.234375 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.5048828125 - response_length_non_aborted/mean:111.234375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.5048828125 - response/aborted_ratio:0.0 - prompt_length/mean:33.0703125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.203578781998658 - timing_s/old_log_prob:8.265849473995331 - timing_s/adv:0.18204157700529322 - timing_s/update_actor:26.25491777499701 - timing_s/update_weights:1.8934964720028802 - timing_s/step:44.96925634500076 - timing_per_token_ms/adv:0.001231941807463681 - timing_per_token_ms/update_actor:0.17767661317062566 - timing_per_token_ms/gen:0.0720218673795359 - perf/total_num_tokens:147768 - perf/time_per_step:44.96925634500076 - perf/throughput:3285.978288507486 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:100 - global_seqlen/min:142224.0 - global_seqlen/max:142224.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142224.0 - global_seqlen/balanced_max:142224.0 - global_seqlen/mean:142224.0 - actor/entropy:0.13420838117599487 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9953261017799377 - training/rollout_probs_diff_mean:0.004548379685729742 - training/rollout_probs_diff_std:0.016198119148612022 - training/rollout_actor_probs_pearson_corr:0.996066153049469 - rollout_corr/training_ppl:1.1538009643554688 - rollout_corr/training_log_ppl:0.14110635221004486 - rollout_corr/kl:0.0008063787245191634 - rollout_corr/k3_kl:0.0010038710897788405 - rollout_corr/rollout_ppl:1.1530190706253052 - rollout_corr/rollout_log_ppl:0.14044331014156342 - rollout_corr/log_ppl_diff:0.0006630202988162637 - rollout_corr/log_ppl_abs_diff:0.0034669709857553244 - rollout_corr/log_ppl_diff_max:0.04567703604698181 - rollout_corr/log_ppl_diff_min:-0.020310699939727783 - rollout_corr/ppl_ratio:1.0006747245788574 - rollout_corr/chi2_token:0.0026273727416992188 - rollout_corr/chi2_seq:0.25314414501190186 - actor/pg_clipfrac:0.0032081501267384738 - actor/ppo_kl:0.0001557119173109337 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.002016969461692497 - actor/loss:-0.0020169697236269712 - actor/grad_norm:1.982421875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.31312131881714 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.515) - val-core/char_count/acc/mean@1:np.float64(0.515) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:100 - training/epoch:1 - critic/score/mean:0.4892578125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4892578125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0017688004299998283 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.0017688004299998283 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:106.734375 - response_length/max:128.0 - response_length/min:32.0 - response_length/clip_ratio:0.4326171875 - response_length_non_aborted/mean:106.734375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:32.0 - response_length_non_aborted/clip_ratio:0.4326171875 - response/aborted_ratio:0.0 - prompt_length/mean:32.15625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.220847717995639 - timing_s/old_log_prob:7.946312248997856 - timing_s/adv:0.18230706699978327 - timing_s/update_actor:25.283638307999354 - timing_s/update_weights:1.9101585559983505 - timing_s/step:43.731934231997 - timing_s/testing:8.894415430004301 - timing_per_token_ms/adv:0.0012818305419604516 - timing_per_token_ms/update_actor:0.17777335968612437 - timing_per_token_ms/gen:0.07521636398400343 - perf/total_num_tokens:142224 - perf/time_per_step:43.731934231997 - perf/throughput:3252.1772132351757 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:101 - global_seqlen/min:143863.0 - global_seqlen/max:143863.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:143863.0 - global_seqlen/balanced_max:143863.0 - global_seqlen/mean:143863.0 - actor/entropy:0.13498355448246002 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8753384351730347 - training/rollout_probs_diff_mean:0.00457461504265666 - training/rollout_probs_diff_std:0.016198042780160904 - training/rollout_actor_probs_pearson_corr:0.9961906671524048 - rollout_corr/training_ppl:1.156746506690979 - rollout_corr/training_log_ppl:0.14337055385112762 - rollout_corr/kl:0.0013433649437502027 - rollout_corr/k3_kl:0.0010663249995559454 - rollout_corr/rollout_ppl:1.1552197933197021 - rollout_corr/rollout_log_ppl:0.1420864462852478 - rollout_corr/log_ppl_diff:0.0012840930139645934 - rollout_corr/log_ppl_abs_diff:0.0037002258468419313 - rollout_corr/log_ppl_diff_max:0.02959725260734558 - rollout_corr/log_ppl_diff_min:-0.017168357968330383 - rollout_corr/ppl_ratio:1.001297116279602 - rollout_corr/chi2_token:0.001698136329650879 - rollout_corr/chi2_seq:0.21406269073486328 - actor/pg_clipfrac:0.0028633245528908446 - actor/ppo_kl:7.866469180347242e-05 - actor/pg_clipfrac_lower:7.922926063959798e-06 - actor/pg_loss:0.0008191941305994987 - actor/loss:0.0008191948290914297 - actor/grad_norm:1.7744140625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.20722484588623 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:101 - training/epoch:1 - critic/score/mean:0.4873046875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4873046875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005861361976712942 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005861361976712942 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.7802734375 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.4345703125 - response_length_non_aborted/mean:107.7802734375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.4345703125 - response/aborted_ratio:0.0 - prompt_length/mean:32.7109375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.20556516799843 - timing_s/old_log_prob:7.9986234409952885 - timing_s/adv:0.383721500998945 - timing_s/update_actor:25.313579007000953 - timing_s/update_weights:1.895694901002571 - timing_s/step:43.96942440800194 - timing_per_token_ms/adv:0.0026672702571122874 - timing_per_token_ms/update_actor:0.17595614582624408 - timing_per_token_ms/gen:0.07434799503473348 - perf/total_num_tokens:143863 - perf/time_per_step:43.96942440800194 - perf/throughput:3271.887270232688 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TransferQueueController pid=2438) 2026-11-01 18:36:42,311 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 28679, Total req/min: 5690.12, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.04 minutes; (TransferQueueController pid=2438) Per-operation statistics: SET_CUSTOM_META: req_count=10708, req/min=2124.54, avg_time=0.000010s, max_time=0.000252s, min_time=0.000003s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.39, avg_time=0.000130s, max_time=0.000214s, min_time=0.000108s; KV_RETRIEVE_META: req_count=10751, req/min=2133.08, avg_time=0.000256s, max_time=0.027294s, min_time=0.000064s; NOTIFY_DATA_UPDATE: req_count=7164, req/min=1421.39, avg_time=0.000083s, max_time=0.000882s, min_time=0.000031s; CLEAR_META: req_count=14, req/min=2.78, avg_time=0.001684s, max_time=0.003886s, min_time=0.000779s; KV_LIST: req_count=35, req/min=6.94, avg_time=0.000244s, max_time=0.000773s, min_time=0.000045s (TaskRunnerV1 pid=2351) Training Progress: 73%|███████▎ | 102/140 [1:21:11<29:03, 45.87s/it] (TaskRunnerV1 pid=2351) Training Progress: 74%|███████▎ | 103/140 [1:21:55<28:01, 45.46s/it] (SimpleStorageUnit pid=2599) 2026-11-01 18:38:07,945 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1103, Total req/min: 209.23, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.27 minutes; (SimpleStorageUnit pid=2599) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.04, avg_time=0.001155s, max_time=0.001943s, min_time=0.000422s; PUT_DATA: req_count=1050, req/min=199.17, avg_time=0.000578s, max_time=0.002316s, min_time=0.000309s; GET_DATA: req_count=37, req/min=7.02, avg_time=0.000632s, max_time=0.001246s, min_time=0.000373s (TaskRunnerV1 pid=2351) Training Progress: 74%|███████▍ | 104/140 [1:22:36<26:31, 44.20s/it] (TaskRunnerV1 pid=2351) step:102 - global_seqlen/min:144406.0 - global_seqlen/max:144406.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:144406.0 - global_seqlen/balanced_max:144406.0 - global_seqlen/mean:144406.0 - actor/entropy:0.13914485275745392 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.681950569152832 - training/rollout_probs_diff_mean:0.004575260449200869 - training/rollout_probs_diff_std:0.01535045262426138 - training/rollout_actor_probs_pearson_corr:0.996566891670227 - rollout_corr/training_ppl:1.161238431930542 - rollout_corr/training_log_ppl:0.14702783524990082 - rollout_corr/kl:0.0010866057127714157 - rollout_corr/k3_kl:0.0009389628539793193 - rollout_corr/rollout_ppl:1.1598601341247559 - rollout_corr/rollout_log_ppl:0.14588911831378937 - rollout_corr/log_ppl_diff:0.001138718449510634 - rollout_corr/log_ppl_abs_diff:0.003451886121183634 - rollout_corr/log_ppl_diff_max:0.03717167675495148 - rollout_corr/log_ppl_diff_min:-0.02381947636604309 - rollout_corr/ppl_ratio:1.001150369644165 - rollout_corr/chi2_token:0.0015355348587036133 - rollout_corr/chi2_seq:0.1852942705154419 - actor/pg_clipfrac:0.0030305133429161897 - actor/ppo_kl:-0.00013512322538862992 - actor/pg_clipfrac_lower:1.7591692691591256e-05 - actor/pg_loss:0.0006452805537264794 - actor/loss:0.0006452817469835281 - actor/grad_norm:1.8671875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.4119348526001 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:102 - training/epoch:1 - critic/score/mean:0.44140625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.44140625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0022490371484309435 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0022490371484309435 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.880859375 - response_length/max:128.0 - response_length/min:38.0 - response_length/clip_ratio:0.4560546875 - response_length_non_aborted/mean:107.880859375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:38.0 - response_length_non_aborted/clip_ratio:0.4560546875 - response/aborted_ratio:0.0 - prompt_length/mean:33.140625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.191404680997948 - timing_s/old_log_prob:8.07848900299723 - timing_s/adv:0.18654786000115564 - timing_s/update_actor:25.59699638500024 - timing_s/update_weights:1.9287343809992308 - timing_s/step:44.15121515100327 - timing_per_token_ms/adv:0.0012918290098829386 - timing_per_token_ms/update_actor:0.1772571526460136 - timing_per_token_ms/gen:0.07415049045892956 - perf/total_num_tokens:144406 - perf/time_per_step:44.15121515100327 - perf/throughput:3270.7140563654134 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:103 - global_seqlen/min:145932.0 - global_seqlen/max:145932.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:145932.0 - global_seqlen/balanced_max:145932.0 - global_seqlen/mean:145932.0 - actor/entropy:0.1324371099472046 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6983710527420044 - training/rollout_probs_diff_mean:0.004451463930308819 - training/rollout_probs_diff_std:0.015347616747021675 - training/rollout_actor_probs_pearson_corr:0.9964416027069092 - rollout_corr/training_ppl:1.1525170803070068 - rollout_corr/training_log_ppl:0.13978278636932373 - rollout_corr/kl:0.0009405725868418813 - rollout_corr/k3_kl:0.0010585524141788483 - rollout_corr/rollout_ppl:1.1513992547988892 - rollout_corr/rollout_log_ppl:0.13885419070720673 - rollout_corr/log_ppl_diff:0.000928606023080647 - rollout_corr/log_ppl_abs_diff:0.0034193615429103374 - rollout_corr/log_ppl_diff_max:0.0953717902302742 - rollout_corr/log_ppl_diff_min:-0.025784343481063843 - rollout_corr/ppl_ratio:1.00094473361969 - rollout_corr/chi2_token:0.002144336700439453 - rollout_corr/chi2_seq:0.21526217460632324 - actor/pg_clipfrac:0.0031668477749917656 - actor/ppo_kl:0.0001862687388438644 - actor/pg_clipfrac_lower:7.57025190978311e-06 - actor/pg_loss:0.0019578142382670194 - actor/loss:0.0019578158389776945 - actor/grad_norm:1.91796875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.2560396194458 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:103 - training/epoch:1 - critic/score/mean:0.48046875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.48046875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008883200585842133 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008883200585842133 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.54296875 - response_length/max:128.0 - response_length/min:43.0 - response_length/clip_ratio:0.4775390625 - response_length_non_aborted/mean:109.54296875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:43.0 - response_length_non_aborted/clip_ratio:0.4775390625 - response/aborted_ratio:0.0 - prompt_length/mean:32.96875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.219803242995113 - timing_s/old_log_prob:8.059743716999947 - timing_s/adv:0.22521639100159518 - timing_s/update_actor:25.60997205899912 - timing_s/update_weights:1.9003940309994505 - timing_s/step:44.19177919399954 - timing_per_token_ms/adv:0.001543296816336343 - timing_per_token_ms/update_actor:0.1754925037620201 - timing_per_token_ms/gen:0.07327856544409579 - perf/total_num_tokens:145932 - perf/time_per_step:44.19177919399954 - perf/throughput:3302.243147065122 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:104 - global_seqlen/min:138453.0 - global_seqlen/max:138453.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:138453.0 - global_seqlen/balanced_max:138453.0 - global_seqlen/mean:138453.0 - actor/entropy:0.13377872109413147 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8710330724716187 - training/rollout_probs_diff_mean:0.004720797762274742 - training/rollout_probs_diff_std:0.016122212633490562 - training/rollout_actor_probs_pearson_corr:0.9962499737739563 - rollout_corr/training_ppl:1.158022403717041 - rollout_corr/training_log_ppl:0.14423735439777374 - rollout_corr/kl:0.0011834234464913607 - rollout_corr/k3_kl:0.0011096748057752848 - rollout_corr/rollout_ppl:1.1565377712249756 - rollout_corr/rollout_log_ppl:0.14301007986068726 - rollout_corr/log_ppl_diff:0.001227260334417224 - rollout_corr/log_ppl_abs_diff:0.004004937596619129 - rollout_corr/log_ppl_diff_max:0.04292592406272888 - rollout_corr/log_ppl_diff_min:-0.017706573009490967 - rollout_corr/ppl_ratio:1.0012433528900146 - rollout_corr/chi2_token:0.002385735511779785 - rollout_corr/chi2_seq:0.2308136224746704 - actor/pg_clipfrac:0.00352634508938839 - actor/ppo_kl:8.923126491329944e-05 - actor/pg_clipfrac_lower:3.38640693371417e-05 - actor/pg_loss:0.0012885350442957133 - actor/loss:0.0012885354808531702 - actor/grad_norm:1.8291015625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.23807430267334 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:104 - training/epoch:1 - critic/score/mean:0.478515625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.478515625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0034703738056123257 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0034703738056123257 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:103.4189453125 - response_length/max:128.0 - response_length/min:30.0 - response_length/clip_ratio:0.34765625 - response_length_non_aborted/mean:103.4189453125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:30.0 - response_length_non_aborted/clip_ratio:0.34765625 - response/aborted_ratio:0.0 - prompt_length/mean:31.7890625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:6.206612858004519 - timing_s/old_log_prob:7.721193388002575 - timing_s/adv:0.18192840199480997 - timing_s/update_actor:24.761049818000174 - timing_s/update_weights:1.901424228999531 - timing_s/step:40.94724260899966 - timing_per_token_ms/adv:0.0013140083782569535 - timing_per_token_ms/update_actor:0.1788408327591325 - timing_per_token_ms/gen:0.05860768886039337 - perf/total_num_tokens:138453 - perf/time_per_step:40.94724260899966 - perf/throughput:3381.2533195964184 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 75%|███████▌ | 105/140 [1:23:30<27:21, 46.90s/it] (SimpleStorageUnit pid=2583) 2026-11-01 18:38:07,946 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1106, Total req/min: 209.80, Total avg process time: 0.0006s; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.27 minutes; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.04, avg_time=0.001333s, max_time=0.004146s, min_time=0.000394s; PUT_DATA: req_count=1053, req/min=199.74, avg_time=0.000595s, max_time=0.002488s, min_time=0.000317s; GET_DATA: req_count=37, req/min=7.02, avg_time=0.000621s, max_time=0.001144s, min_time=0.000412s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 76%|███████▌ | 106/140 [1:24:14<26:11, 46.22s/it] (TaskRunnerV1 pid=2351) Training Progress: 76%|███████▋ | 107/140 [1:24:58<25:00, 45.48s/it] (TaskRunnerV1 pid=2351) step:105 - global_seqlen/min:146263.0 - global_seqlen/max:146263.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146263.0 - global_seqlen/balanced_max:146263.0 - global_seqlen/mean:146263.0 - actor/entropy:0.13140910863876343 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9200974702835083 - training/rollout_probs_diff_mean:0.004404248669743538 - training/rollout_probs_diff_std:0.01607844978570938 - training/rollout_actor_probs_pearson_corr:0.9961610436439514 - rollout_corr/training_ppl:1.1521764993667603 - rollout_corr/training_log_ppl:0.13940301537513733 - rollout_corr/kl:0.001094728708267212 - rollout_corr/k3_kl:0.0010045969393104315 - rollout_corr/rollout_ppl:1.1507469415664673 - rollout_corr/rollout_log_ppl:0.13820411264896393 - rollout_corr/log_ppl_diff:0.0011989151826128364 - rollout_corr/log_ppl_abs_diff:0.003539824392646551 - rollout_corr/log_ppl_diff_max:0.03871607780456543 - rollout_corr/log_ppl_diff_min:-0.013773232698440552 - rollout_corr/ppl_ratio:1.0012117624282837 - rollout_corr/chi2_token:0.0026140213012695312 - rollout_corr/chi2_seq:0.18630635738372803 - actor/pg_clipfrac:0.003583366284146905 - actor/ppo_kl:0.00016803319530348895 - actor/pg_clipfrac_lower:1.5327894895259913e-05 - actor/pg_loss:-0.00011107642785646021 - actor/loss:-0.00011107709724456072 - actor/grad_norm:1.732421875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.220282554626465 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.515) - val-core/char_count/acc/mean@1:np.float64(0.515) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:105 - training/epoch:1 - critic/score/mean:0.4482421875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4482421875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.003770554903894663 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.003770554903894663 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.6396484375 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.4833984375 - response_length_non_aborted/mean:109.6396484375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.4833984375 - response/aborted_ratio:0.0 - prompt_length/mean:33.1953125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.189085061996593 - timing_s/old_log_prob:8.076851921003254 - timing_s/adv:0.18078949500340968 - timing_s/update_actor:25.682510871003615 - timing_s/update_weights:1.8939511069984292 - timing_s/step:44.194824427999265 - timing_s/testing:8.887050579003699 - timing_per_token_ms/adv:0.0012360576154147643 - timing_per_token_ms/update_actor:0.17559130382259092 - timing_per_token_ms/gen:0.07294034133477562 - perf/total_num_tokens:146263 - perf/time_per_step:44.194824427999265 - perf/throughput:3309.5051715452973 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:106 - global_seqlen/min:146926.0 - global_seqlen/max:146926.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:146926.0 - global_seqlen/balanced_max:146926.0 - global_seqlen/mean:146926.0 - actor/entropy:0.1338597983121872 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9104164242744446 - training/rollout_probs_diff_mean:0.004681644961237907 - training/rollout_probs_diff_std:0.01682807318866253 - training/rollout_actor_probs_pearson_corr:0.9958949089050293 - rollout_corr/training_ppl:1.1566376686096191 - rollout_corr/training_log_ppl:0.1429755836725235 - rollout_corr/kl:0.001444556168280542 - rollout_corr/k3_kl:0.001132275676354766 - rollout_corr/rollout_ppl:1.1548895835876465 - rollout_corr/rollout_log_ppl:0.14150603115558624 - rollout_corr/log_ppl_diff:0.0014695549616590142 - rollout_corr/log_ppl_abs_diff:0.003764715977013111 - rollout_corr/log_ppl_diff_max:0.052617087960243225 - rollout_corr/log_ppl_diff_min:-0.016512110829353333 - rollout_corr/ppl_ratio:1.0014865398406982 - rollout_corr/chi2_token:0.0012992620468139648 - rollout_corr/chi2_seq:0.19814324378967285 - actor/pg_clipfrac:0.0028698881214950234 - actor/ppo_kl:-0.00010781611649690603 - actor/pg_clipfrac_lower:5.693965310153241e-05 - actor/pg_loss:0.0006284057744778693 - actor/loss:0.0006284050177782774 - actor/grad_norm:1.74609375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.37851047515869 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:106 - training/epoch:1 - critic/score/mean:0.392578125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.392578125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008587530814111233 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008587530814111233 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:109.599609375 - response_length/max:128.0 - response_length/min:34.0 - response_length/clip_ratio:0.50390625 - response_length_non_aborted/mean:109.599609375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:34.0 - response_length_non_aborted/clip_ratio:0.50390625 - response/aborted_ratio:0.0 - prompt_length/mean:33.8828125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.1171875 - timing_s/gen:8.193560675994377 - timing_s/old_log_prob:8.120496856994578 - timing_s/adv:0.18300472899863962 - timing_s/update_actor:25.93071131799661 - timing_s/update_weights:1.8924426750018029 - timing_s/step:44.49235337400023 - timing_per_token_ms/adv:0.0012455571444035748 - timing_per_token_ms/update_actor:0.17648824114177622 - timing_per_token_ms/gen:0.07300686693392476 - perf/total_num_tokens:146926 - perf/time_per_step:44.49235337400023 - perf/throughput:3302.2753093087317 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:107 - global_seqlen/min:142062.0 - global_seqlen/max:142062.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142062.0 - global_seqlen/balanced_max:142062.0 - global_seqlen/mean:142062.0 - actor/entropy:0.13006986677646637 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9810768365859985 - training/rollout_probs_diff_mean:0.00443977490067482 - training/rollout_probs_diff_std:0.01685478165745735 - training/rollout_actor_probs_pearson_corr:0.9956957697868347 - rollout_corr/training_ppl:1.152018427848816 - rollout_corr/training_log_ppl:0.13908056914806366 - rollout_corr/kl:0.0017777185421437025 - rollout_corr/k3_kl:0.0014210023218765855 - rollout_corr/rollout_ppl:1.1493395566940308 - rollout_corr/rollout_log_ppl:0.13692256808280945 - rollout_corr/log_ppl_diff:0.0021579957101494074 - rollout_corr/log_ppl_abs_diff:0.004438282921910286 - rollout_corr/log_ppl_diff_max:0.1718987673521042 - rollout_corr/log_ppl_diff_min:-0.02316504716873169 - rollout_corr/ppl_ratio:1.0022526979446411 - rollout_corr/chi2_token:0.0009642839431762695 - rollout_corr/chi2_seq:0.1130068302154541 - actor/pg_clipfrac:0.0029359299830199843 - actor/ppo_kl:-0.0003421844080264691 - actor/pg_clipfrac_lower:1.8014758097706363e-05 - actor/pg_loss:3.758614184334874e-05 - actor/loss:3.758375532925129e-05 - actor/grad_norm:2.0517578125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.32043695449829 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:107 - training/epoch:1 - critic/score/mean:0.525390625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.525390625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008034276776015759 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008034276776015759 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:106.029296875 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.4052734375 - response_length_non_aborted/mean:106.029296875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.4052734375 - response/aborted_ratio:0.0 - prompt_length/mean:32.703125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.193101880999166 - timing_s/old_log_prob:7.909762104995025 - timing_s/adv:0.24123833799967542 - timing_s/update_actor:25.023819846996048 - timing_s/update_weights:1.917950186994858 - timing_s/step:43.46028922199912 - timing_per_token_ms/adv:0.0016981201024881772 - timing_per_token_ms/update_actor:0.17614717410001302 - timing_per_token_ms/gen:0.07546099324883643 - perf/total_num_tokens:142062 - perf/time_per_step:43.46028922199912 - perf/throughput:3268.7771421477282 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 77%|███████▋ | 108/140 [1:25:42<24:03, 45.11s/it] (TransferQueueController pid=2438) 2026-11-01 18:41:57,726 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 32275, Total req/min: 6139.53, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.26 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=12035, req/min=2289.37, avg_time=0.000258s, max_time=0.065036s, min_time=0.000063s; CLEAR_META: req_count=16, req/min=3.04, avg_time=0.001633s, max_time=0.002341s, min_time=0.000806s; SET_CUSTOM_META: req_count=11989, req/min=2280.61, avg_time=0.000009s, max_time=0.000263s, min_time=0.000003s; KV_LIST: req_count=39, req/min=7.42, avg_time=0.000248s, max_time=0.000834s, min_time=0.000045s; NOTIFY_DATA_UPDATE: req_count=8189, req/min=1557.76, avg_time=0.000083s, max_time=0.001465s, min_time=0.000032s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.33, avg_time=0.000134s, max_time=0.000226s, min_time=0.000108s (TaskRunnerV1 pid=2351) Training Progress: 78%|███████▊ | 109/140 [1:26:26<23:05, 44.71s/it] (TaskRunnerV1 pid=2351) Training Progress: 79%|███████▊ | 110/140 [1:27:18<23:24, 46.83s/it] (TaskRunnerV1 pid=2351) step:108 - global_seqlen/min:145568.0 - global_seqlen/max:145568.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:145568.0 - global_seqlen/balanced_max:145568.0 - global_seqlen/mean:145568.0 - actor/entropy:0.12896792590618134 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.999943196773529 - training/rollout_probs_diff_mean:0.004471893422305584 - training/rollout_probs_diff_std:0.0161801278591156 - training/rollout_actor_probs_pearson_corr:0.9960744976997375 - rollout_corr/training_ppl:1.1508253812789917 - rollout_corr/training_log_ppl:0.13837212324142456 - rollout_corr/kl:0.0011680618626996875 - rollout_corr/k3_kl:0.00113074597902596 - rollout_corr/rollout_ppl:1.1494919061660767 - rollout_corr/rollout_log_ppl:0.13724824786186218 - rollout_corr/log_ppl_diff:0.0011238673469051719 - rollout_corr/log_ppl_abs_diff:0.003592154011130333 - rollout_corr/log_ppl_diff_max:0.08002415299415588 - rollout_corr/log_ppl_diff_min:-0.02155175805091858 - rollout_corr/ppl_ratio:1.0011389255523682 - rollout_corr/chi2_token:0.003059983253479004 - rollout_corr/chi2_seq:0.4313957691192627 - actor/pg_clipfrac:0.003724234717083163 - actor/ppo_kl:-0.000207624387385863 - actor/pg_clipfrac_lower:3.19736506450378e-05 - actor/pg_loss:0.0013187707227189094 - actor/loss:0.0013187729055061936 - actor/grad_norm:2.0302734375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.25004482269287 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:108 - training/epoch:1 - critic/score/mean:0.4765625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4765625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.002691258443519473 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.002691258443519473 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:108.984375 - response_length/max:128.0 - response_length/min:28.0 - response_length/clip_ratio:0.421875 - response_length_non_aborted/mean:108.984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:28.0 - response_length_non_aborted/clip_ratio:0.421875 - response/aborted_ratio:0.0 - prompt_length/mean:33.171875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.189455959000043 - timing_s/old_log_prob:8.079232625001168 - timing_s/adv:0.18535969500226201 - timing_s/update_actor:25.586317897999834 - timing_s/update_weights:1.8952272210008232 - timing_s/step:44.10169453800336 - timing_per_token_ms/adv:0.001273354686485093 - timing_per_token_ms/update_actor:0.17576883585678058 - timing_per_token_ms/gen:0.07338222185483909 - perf/total_num_tokens:145568 - perf/time_per_step:44.10169453800336 - perf/throughput:3300.7348476045745 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:109 - global_seqlen/min:142627.0 - global_seqlen/max:142627.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142627.0 - global_seqlen/balanced_max:142627.0 - global_seqlen/mean:142627.0 - actor/entropy:0.12853692471981049 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8093011975288391 - training/rollout_probs_diff_mean:0.00431904336437583 - training/rollout_probs_diff_std:0.015044093132019043 - training/rollout_actor_probs_pearson_corr:0.9964391589164734 - rollout_corr/training_ppl:1.1446638107299805 - rollout_corr/training_log_ppl:0.1331973671913147 - rollout_corr/kl:0.0008340274798683822 - rollout_corr/k3_kl:0.0009328256128355861 - rollout_corr/rollout_ppl:1.1436643600463867 - rollout_corr/rollout_log_ppl:0.13234686851501465 - rollout_corr/log_ppl_diff:0.0008504833676852286 - rollout_corr/log_ppl_abs_diff:0.0033430131152272224 - rollout_corr/log_ppl_diff_max:0.022440895438194275 - rollout_corr/log_ppl_diff_min:-0.022434309124946594 - rollout_corr/ppl_ratio:1.0008610486984253 - rollout_corr/chi2_token:0.0028475522994995117 - rollout_corr/chi2_seq:0.3738830089569092 - actor/pg_clipfrac:0.003176040755837069 - actor/ppo_kl:0.00017923053527132046 - actor/pg_clipfrac_lower:7.85274520846239e-06 - actor/pg_loss:-0.000171952648088336 - actor/loss:-0.00017195171676576138 - actor/grad_norm:1.6318359375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.243563652038574 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:109 - training/epoch:1 - critic/score/mean:0.5166015625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5166015625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.005729961208999157 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.005729961208999157 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.0732421875 - response_length/max:128.0 - response_length/min:33.0 - response_length/clip_ratio:0.4033203125 - response_length_non_aborted/mean:107.0732421875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:33.0 - response_length_non_aborted/clip_ratio:0.4033203125 - response/aborted_ratio:0.0 - prompt_length/mean:32.2109375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.195252941004583 - timing_s/old_log_prob:7.999700458000007 - timing_s/adv:0.18803147600556258 - timing_s/update_actor:25.18548481199832 - timing_s/update_weights:1.8928874490011367 - timing_s/step:43.63085559899628 - timing_per_token_ms/adv:0.0013183441845202 - timing_per_token_ms/update_actor:0.17658286868544049 - timing_per_token_ms/gen:0.07474488057609316 - perf/total_num_tokens:142627 - perf/time_per_step:43.63085559899628 - perf/throughput:3268.9480424326384 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:110 - global_seqlen/min:142677.0 - global_seqlen/max:142677.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142677.0 - global_seqlen/balanced_max:142677.0 - global_seqlen/mean:142677.0 - actor/entropy:0.1278965175151825 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6580967903137207 - training/rollout_probs_diff_mean:0.0044449083507061005 - training/rollout_probs_diff_std:0.015185588970780373 - training/rollout_actor_probs_pearson_corr:0.9964361190795898 - rollout_corr/training_ppl:1.1455860137939453 - rollout_corr/training_log_ppl:0.1340862661600113 - rollout_corr/kl:0.0009314167546108365 - rollout_corr/k3_kl:0.000939690915402025 - rollout_corr/rollout_ppl:1.1444602012634277 - rollout_corr/rollout_log_ppl:0.13312475383281708 - rollout_corr/log_ppl_diff:0.0009615127928555012 - rollout_corr/log_ppl_abs_diff:0.003450787626206875 - rollout_corr/log_ppl_diff_max:0.02576442062854767 - rollout_corr/log_ppl_diff_min:-0.026050463318824768 - rollout_corr/ppl_ratio:1.0009726285934448 - rollout_corr/chi2_token:0.002041935920715332 - rollout_corr/chi2_seq:0.2629387378692627 - actor/pg_clipfrac:0.0031287232680673087 - actor/ppo_kl:2.9111489917947132e-05 - actor/pg_clipfrac_lower:7.574380409399358e-06 - actor/pg_loss:0.00030235864687711 - actor/loss:0.0003023585304617882 - actor/grad_norm:2.1767578125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.23013973236084 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.524) - val-core/char_count/acc/mean@1:np.float64(0.524) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:110 - training/epoch:1 - critic/score/mean:0.482421875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.482421875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00023519490787293762 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.00023519490787293762 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:106.9345703125 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.419921875 - response_length_non_aborted/mean:106.9345703125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.419921875 - response/aborted_ratio:0.0 - prompt_length/mean:32.3984375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.205658338003559 - timing_s/old_log_prob:8.156538558003376 - timing_s/adv:0.19655388899991522 - timing_s/update_actor:25.16866265200224 - timing_s/update_weights:1.8947689139968134 - timing_s/step:43.790967328997795 - timing_s/testing:7.63165151100111 - timing_per_token_ms/adv:0.0013776143947511878 - timing_per_token_ms/update_actor:0.17640308285149142 - timing_per_token_ms/gen:0.07493683471387073 - perf/total_num_tokens:142677 - perf/time_per_step:43.790967328997795 - perf/throughput:3258.1376640547787 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(SimpleStorageUnit pid=2599) 2026-11-01 18:43:33,198 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1243, Total req/min: 229.30, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.42 minutes; (SimpleStorageUnit pid=2599) Per-operation statistics: CLEAR_DATA: req_count=18, req/min=3.32, avg_time=0.001460s, max_time=0.003902s, min_time=0.000427s; PUT_DATA: req_count=1186, req/min=218.78, avg_time=0.000612s, max_time=0.002201s, min_time=0.000312s; GET_DATA: req_count=39, req/min=7.19, avg_time=0.000688s, max_time=0.001245s, min_time=0.000407s (TaskRunnerV1 pid=2351) Training Progress: 79%|███████▉ | 111/140 [1:28:01<22:10, 45.88s/it] (TaskRunnerV1 pid=2351) Training Progress: 80%|████████ | 112/140 [1:28:46<21:13, 45.48s/it] (SimpleStorageUnit pid=2583) 2026-11-01 18:43:33,197 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1218, Total req/min: 224.69, Total avg process time: 0.0006s; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.42 minutes; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=18, req/min=3.32, avg_time=0.001472s, max_time=0.004246s, min_time=0.000422s; PUT_DATA: req_count=1161, req/min=214.17, avg_time=0.000576s, max_time=0.002550s, min_time=0.000304s; GET_DATA: req_count=39, req/min=7.19, avg_time=0.000614s, max_time=0.001174s, min_time=0.000407s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 81%|████████ | 113/140 [1:29:30<20:12, 44.90s/it] (TaskRunnerV1 pid=2351) step:111 - global_seqlen/min:142551.0 - global_seqlen/max:142551.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142551.0 - global_seqlen/balanced_max:142551.0 - global_seqlen/mean:142551.0 - actor/entropy:0.12882405519485474 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8432993292808533 - training/rollout_probs_diff_mean:0.00439003761857748 - training/rollout_probs_diff_std:0.015596121549606323 - training/rollout_actor_probs_pearson_corr:0.9963210225105286 - rollout_corr/training_ppl:1.1485490798950195 - rollout_corr/training_log_ppl:0.13664034008979797 - rollout_corr/kl:0.0010450457921251655 - rollout_corr/k3_kl:0.001017191680148244 - rollout_corr/rollout_ppl:1.1473822593688965 - rollout_corr/rollout_log_ppl:0.13565784692764282 - rollout_corr/log_ppl_diff:0.0009824687149375677 - rollout_corr/log_ppl_abs_diff:0.0036363848485052586 - rollout_corr/log_ppl_diff_max:0.08119021356105804 - rollout_corr/log_ppl_diff_min:-0.017474502325057983 - rollout_corr/ppl_ratio:1.0009973049163818 - rollout_corr/chi2_token:0.0018024444580078125 - rollout_corr/chi2_seq:0.2250218391418457 - actor/pg_clipfrac:0.003105246309132781 - actor/ppo_kl:-5.821675555732025e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0023609897325513884 - actor/loss:0.002360990736633539 - actor/grad_norm:1.7978515625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.30009984970093 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:111 - training/epoch:1 - critic/score/mean:0.4677734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4677734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007180938497185707 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007180938497185707 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:106.7490234375 - response_length/max:128.0 - response_length/min:39.0 - response_length/clip_ratio:0.3876953125 - response_length_non_aborted/mean:106.7490234375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:39.0 - response_length_non_aborted/clip_ratio:0.3876953125 - response/aborted_ratio:0.0 - prompt_length/mean:32.4609375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.206012383998313 - timing_s/old_log_prob:7.960332165996078 - timing_s/adv:0.2439204249967588 - timing_s/update_actor:25.0665535549997 - timing_s/update_weights:1.9106792500024312 - timing_s/step:43.557732567998755 - timing_per_token_ms/adv:0.0017111098834575612 - timing_per_token_ms/update_actor:0.17584270580353487 - timing_per_token_ms/gen:0.07507032580434095 - perf/total_num_tokens:142551 - perf/time_per_step:43.557732567998755 - perf/throughput:3272.6910147919452 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:112 - global_seqlen/min:142942.0 - global_seqlen/max:142942.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142942.0 - global_seqlen/balanced_max:142942.0 - global_seqlen/mean:142942.0 - actor/entropy:0.13108482956886292 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9957264065742493 - training/rollout_probs_diff_mean:0.004423810169100761 - training/rollout_probs_diff_std:0.015383795835077763 - training/rollout_actor_probs_pearson_corr:0.9965100288391113 - rollout_corr/training_ppl:1.1498973369598389 - rollout_corr/training_log_ppl:0.13753756880760193 - rollout_corr/kl:0.0009460086585022509 - rollout_corr/k3_kl:0.001039291499182582 - rollout_corr/rollout_ppl:1.1488924026489258 - rollout_corr/rollout_log_ppl:0.13667228817939758 - rollout_corr/log_ppl_diff:0.0008652644464746118 - rollout_corr/log_ppl_abs_diff:0.003331074956804514 - rollout_corr/log_ppl_diff_max:0.04616905003786087 - rollout_corr/log_ppl_diff_min:-0.017440542578697205 - rollout_corr/ppl_ratio:1.0008773803710938 - rollout_corr/chi2_token:0.0026291608810424805 - rollout_corr/chi2_seq:0.19117724895477295 - actor/pg_clipfrac:0.003299941890873015 - actor/ppo_kl:0.0001918232511949706 - actor/pg_clipfrac_lower:1.6417331911119014e-05 - actor/pg_loss:0.0018603440839797258 - actor/loss:0.0018603447824716568 - actor/grad_norm:1.970703125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.240195751190186 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:112 - training/epoch:1 - critic/score/mean:0.4697265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4697265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0006061292951926589 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.0006061292951926589 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.123046875 - response_length/max:128.0 - response_length/min:35.0 - response_length/clip_ratio:0.423828125 - response_length_non_aborted/mean:107.123046875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:35.0 - response_length_non_aborted/clip_ratio:0.423828125 - response/aborted_ratio:0.0 - prompt_length/mean:32.46875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0234375 - timing_s/gen:8.198845803999575 - timing_s/old_log_prob:7.996603170002345 - timing_s/adv:0.1815411130010034 - timing_s/update_actor:25.773182424003608 - timing_s/update_weights:1.899055560999841 - timing_s/step:44.39612599300017 - timing_per_token_ms/adv:0.0012700333911726672 - timing_per_token_ms/update_actor:0.18030517569366322 - timing_per_token_ms/gen:0.07474288296533607 - perf/total_num_tokens:142942 - perf/time_per_step:44.39612599300017 - perf/throughput:3219.6953405920444 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:113 - global_seqlen/min:141268.0 - global_seqlen/max:141268.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:141268.0 - global_seqlen/balanced_max:141268.0 - global_seqlen/mean:141268.0 - actor/entropy:0.13303738832473755 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9382734894752502 - training/rollout_probs_diff_mean:0.004600347485393286 - training/rollout_probs_diff_std:0.017337720841169357 - training/rollout_actor_probs_pearson_corr:0.9955093860626221 - rollout_corr/training_ppl:1.1513495445251465 - rollout_corr/training_log_ppl:0.13893680274486542 - rollout_corr/kl:0.0011156623950228095 - rollout_corr/k3_kl:0.0011594019597396255 - rollout_corr/rollout_ppl:1.1499913930892944 - rollout_corr/rollout_log_ppl:0.13779860734939575 - rollout_corr/log_ppl_diff:0.0011381669901311398 - rollout_corr/log_ppl_abs_diff:0.0037125747185200453 - rollout_corr/log_ppl_diff_max:0.04819931089878082 - rollout_corr/log_ppl_diff_min:-0.02057620882987976 - rollout_corr/ppl_ratio:1.0011543035507202 - rollout_corr/chi2_token:0.0024385452270507812 - rollout_corr/chi2_seq:0.2740662097930908 - actor/pg_clipfrac:0.0031065426058679195 - actor/ppo_kl:0.0001383049811162588 - actor/pg_clipfrac_lower:3.919013306585839e-05 - actor/pg_loss:0.0007462352514266968 - actor/loss:0.0007462349021807313 - actor/grad_norm:2.083984375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.30444097518921 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:113 - training/epoch:1 - critic/score/mean:0.4599609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4599609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0014437987701967359 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0014437987701967359 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:105.59765625 - response_length/max:128.0 - response_length/min:33.0 - response_length/clip_ratio:0.4130859375 - response_length_non_aborted/mean:105.59765625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:33.0 - response_length_non_aborted/clip_ratio:0.4130859375 - response/aborted_ratio:0.0 - prompt_length/mean:32.359375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:8.183130930003244 - timing_s/old_log_prob:7.891856149995874 - timing_s/adv:0.1828434510025545 - timing_s/update_actor:24.8842919000017 - timing_s/update_weights:1.9283202590013389 - timing_s/step:43.24710635199881 - timing_per_token_ms/adv:0.0012943019721561465 - timing_per_token_ms/update_actor:0.1761495306792883 - timing_per_token_ms/gen:0.07567723643327826 - perf/total_num_tokens:141268 - perf/time_per_step:43.24710635199881 - perf/throughput:3266.5306864737977 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 81%|████████▏ | 114/140 [1:30:13<19:18, 44.56s/it] (TaskRunnerV1 pid=2351) Training Progress: 82%|████████▏ | 115/140 [1:31:06<19:37, 47.12s/it] (TransferQueueController pid=2438) 2026-11-01 18:47:21,630 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 39288, Total req/min: 7277.73, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.40 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=15040, req/min=2786.02, avg_time=0.000231s, max_time=0.016618s, min_time=0.000062s; CLEAR_META: req_count=18, req/min=3.33, avg_time=0.001708s, max_time=0.004040s, min_time=0.000812s; SET_CUSTOM_META: req_count=14990, req/min=2776.76, avg_time=0.000009s, max_time=0.000223s, min_time=0.000003s; KV_LIST: req_count=44, req/min=8.15, avg_time=0.000256s, max_time=0.001033s, min_time=0.000050s; NOTIFY_DATA_UPDATE: req_count=9189, req/min=1702.18, avg_time=0.000081s, max_time=0.000650s, min_time=0.000030s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.30, avg_time=0.000150s, max_time=0.000258s, min_time=0.000109s (TaskRunnerV1 pid=2351) Training Progress: 83%|████████▎ | 116/140 [1:31:50<18:24, 46.04s/it] (TaskRunnerV1 pid=2351) step:114 - global_seqlen/min:142663.0 - global_seqlen/max:142663.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142663.0 - global_seqlen/balanced_max:142663.0 - global_seqlen/mean:142663.0 - actor/entropy:0.13105687499046326 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9295967817306519 - training/rollout_probs_diff_mean:0.004495106637477875 - training/rollout_probs_diff_std:0.01626681163907051 - training/rollout_actor_probs_pearson_corr:0.9960388541221619 - rollout_corr/training_ppl:1.1502279043197632 - rollout_corr/training_log_ppl:0.13806214928627014 - rollout_corr/kl:0.001251248293556273 - rollout_corr/k3_kl:0.0010146762942895293 - rollout_corr/rollout_ppl:1.1488240957260132 - rollout_corr/rollout_log_ppl:0.1368687003850937 - rollout_corr/log_ppl_diff:0.00119345064740628 - rollout_corr/log_ppl_abs_diff:0.0035906299017369747 - rollout_corr/log_ppl_diff_max:0.045085370540618896 - rollout_corr/log_ppl_diff_min:-0.030939847230911255 - rollout_corr/ppl_ratio:1.0012073516845703 - rollout_corr/chi2_token:0.0013606548309326172 - rollout_corr/chi2_seq:0.1402590274810791 - actor/pg_clipfrac:0.0028893253329442814 - actor/ppo_kl:8.44166776611625e-05 - actor/pg_clipfrac_lower:1.7649271285336e-05 - actor/pg_loss:-0.00016596721252426505 - actor/loss:-0.00016596843488514423 - actor/grad_norm:2.0107421875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.31233263015747 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:114 - training/epoch:1 - critic/score/mean:0.5205078125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5205078125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0034570847637951374 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.0034570847637951374 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:106.7568359375 - response_length/max:128.0 - response_length/min:26.0 - response_length/clip_ratio:0.3994140625 - response_length_non_aborted/mean:106.7568359375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:26.0 - response_length_non_aborted/clip_ratio:0.3994140625 - response/aborted_ratio:0.0 - prompt_length/mean:32.5625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.193810085002042 - timing_s/old_log_prob:7.984488466005132 - timing_s/adv:0.18451482300588395 - timing_s/update_actor:25.190334134000295 - timing_s/update_weights:1.917839672001719 - timing_s/step:43.64235782600008 - timing_per_token_ms/adv:0.001293361439237111 - timing_per_token_ms/update_actor:0.17657230069464608 - timing_per_token_ms/gen:0.07495321110696257 - perf/total_num_tokens:142663 - perf/time_per_step:43.64235782600008 - perf/throughput:3268.9113766215455 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:115 - global_seqlen/min:143779.0 - global_seqlen/max:143779.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:143779.0 - global_seqlen/balanced_max:143779.0 - global_seqlen/mean:143779.0 - actor/entropy:0.1290639042854309 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8445901870727539 - training/rollout_probs_diff_mean:0.0043525900691747665 - training/rollout_probs_diff_std:0.015151210129261017 - training/rollout_actor_probs_pearson_corr:0.9964795708656311 - rollout_corr/training_ppl:1.1492998600006104 - rollout_corr/training_log_ppl:0.136976957321167 - rollout_corr/kl:0.0008950986666604877 - rollout_corr/k3_kl:0.0009204670204780996 - rollout_corr/rollout_ppl:1.1481595039367676 - rollout_corr/rollout_log_ppl:0.13602645695209503 - rollout_corr/log_ppl_diff:0.0009505141642875969 - rollout_corr/log_ppl_abs_diff:0.003315421985462308 - rollout_corr/log_ppl_diff_max:0.03560926020145416 - rollout_corr/log_ppl_diff_min:-0.02745164930820465 - rollout_corr/ppl_ratio:1.0009621381759644 - rollout_corr/chi2_token:0.0017648935317993164 - rollout_corr/chi2_seq:0.18585479259490967 - actor/pg_clipfrac:0.0027272288255820363 - actor/ppo_kl:-0.00010908036754623633 - actor/pg_clipfrac_lower:1.6385726970232405e-05 - actor/pg_loss:0.001726182788843289 - actor/loss:0.0017261832545045763 - actor/grad_norm:1.986328125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.28154802322388 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.532) - val-core/char_count/acc/mean@1:np.float64(0.532) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:115 - training/epoch:1 - critic/score/mean:0.48046875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.48046875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008230487816035748 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008230487816035748 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.7060546875 - response_length/max:128.0 - response_length/min:34.0 - response_length/clip_ratio:0.423828125 - response_length_non_aborted/mean:107.7060546875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:34.0 - response_length_non_aborted/clip_ratio:0.423828125 - response/aborted_ratio:0.0 - prompt_length/mean:32.703125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.192449265996402 - timing_s/old_log_prob:8.213252118999662 - timing_s/adv:0.18847836799977813 - timing_s/update_actor:25.373804057002417 - timing_s/update_weights:1.921949660005339 - timing_s/step:44.057207727993955 - timing_s/testing:8.889122230000794 - timing_per_token_ms/adv:0.0013108894066572875 - timing_per_token_ms/update_actor:0.17647781704562152 - timing_per_token_ms/gen:0.07428030633502644 - perf/total_num_tokens:143779 - perf/time_per_step:44.057207727993955 - perf/throughput:3263.461472358422 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:116 - global_seqlen/min:139846.0 - global_seqlen/max:139846.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:139846.0 - global_seqlen/balanced_max:139846.0 - global_seqlen/mean:139846.0 - actor/entropy:0.13201837241649628 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8287922143936157 - training/rollout_probs_diff_mean:0.004556942265480757 - training/rollout_probs_diff_std:0.01563122682273388 - training/rollout_actor_probs_pearson_corr:0.9963714480400085 - rollout_corr/training_ppl:1.1501249074935913 - rollout_corr/training_log_ppl:0.13793882727622986 - rollout_corr/kl:0.0012248811544850469 - rollout_corr/k3_kl:0.0009264874388463795 - rollout_corr/rollout_ppl:1.1486060619354248 - rollout_corr/rollout_log_ppl:0.13664907217025757 - rollout_corr/log_ppl_diff:0.0012897502165287733 - rollout_corr/log_ppl_abs_diff:0.0036075927782803774 - rollout_corr/log_ppl_diff_max:0.02971474826335907 - rollout_corr/log_ppl_diff_min:-0.0209883451461792 - rollout_corr/ppl_ratio:1.0013017654418945 - rollout_corr/chi2_token:0.0013333559036254883 - rollout_corr/chi2_seq:0.13544273376464844 - actor/pg_clipfrac:0.0032635068103748686 - actor/ppo_kl:-9.103907261002558e-05 - actor/pg_clipfrac_lower:7.75193802837748e-06 - actor/pg_loss:0.0007511452859034762 - actor/loss:0.0007511461153626442 - actor/grad_norm:1.931640625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.261435985565186 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:116 - training/epoch:1 - critic/score/mean:0.5185546875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5185546875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.001508631044998765 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.001508631044998765 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.755859375 - response_length/max:128.0 - response_length/min:41.0 - response_length/clip_ratio:0.359375 - response_length_non_aborted/mean:104.755859375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:41.0 - response_length_non_aborted/clip_ratio:0.359375 - response/aborted_ratio:0.0 - prompt_length/mean:31.8125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0390625 - timing_s/gen:8.21001928899932 - timing_s/old_log_prob:7.953990513000463 - timing_s/adv:0.1852531109980191 - timing_s/update_actor:24.94628091499908 - timing_s/update_weights:1.9290041360000032 - timing_s/step:43.391438209000626 - timing_per_token_ms/adv:0.0013246936701658905 - timing_per_token_ms/update_actor:0.17838394315889677 - timing_per_token_ms/gen:0.07653602394890761 - perf/total_num_tokens:139846 - perf/time_per_step:43.391438209000626 - perf/throughput:3222.893865061885 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 84%|████████▎ | 117/140 [1:32:32<17:10, 44.81s/it] (SimpleStorageUnit pid=2599) 2026-11-01 18:48:47,035 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1094, Total req/min: 209.15, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.23 minutes; (SimpleStorageUnit pid=2599) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.06, avg_time=0.001303s, max_time=0.002778s, min_time=0.000428s; PUT_DATA: req_count=1041, req/min=199.02, avg_time=0.000597s, max_time=0.001331s, min_time=0.000307s; GET_DATA: req_count=37, req/min=7.07, avg_time=0.000657s, max_time=0.001211s, min_time=0.000409s (TaskRunnerV1 pid=2351) Training Progress: 84%|████████▍ | 118/140 [1:33:15<16:16, 44.40s/it] (TaskRunnerV1 pid=2351) Training Progress: 85%|████████▌ | 119/140 [1:33:59<15:26, 44.13s/it] (SimpleStorageUnit pid=2583) 2026-11-01 18:48:47,035 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1090, Total req/min: 208.39, Total avg process time: 0.0006s; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.23 minutes; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.06, avg_time=0.001643s, max_time=0.004148s, min_time=0.000402s; PUT_DATA: req_count=1037, req/min=198.25, avg_time=0.000609s, max_time=0.002078s, min_time=0.000318s; GET_DATA: req_count=37, req/min=7.07, avg_time=0.000655s, max_time=0.001297s, min_time=0.000405s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) step:117 - global_seqlen/min:141028.0 - global_seqlen/max:141028.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:141028.0 - global_seqlen/balanced_max:141028.0 - global_seqlen/mean:141028.0 - actor/entropy:0.13203908503055573 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.844563901424408 - training/rollout_probs_diff_mean:0.004380608908832073 - training/rollout_probs_diff_std:0.015047446824610233 - training/rollout_actor_probs_pearson_corr:0.9966013431549072 - rollout_corr/training_ppl:1.1497775316238403 - rollout_corr/training_log_ppl:0.13764825463294983 - rollout_corr/kl:0.0009708324214443564 - rollout_corr/k3_kl:0.0008531904313713312 - rollout_corr/rollout_ppl:1.1486345529556274 - rollout_corr/rollout_log_ppl:0.13667888939380646 - rollout_corr/log_ppl_diff:0.0009693472529761493 - rollout_corr/log_ppl_abs_diff:0.0034213669132441282 - rollout_corr/log_ppl_diff_max:0.022492356598377228 - rollout_corr/log_ppl_diff_min:-0.01665528118610382 - rollout_corr/ppl_ratio:1.0009801387786865 - rollout_corr/chi2_token:0.0014438629150390625 - rollout_corr/chi2_seq:0.17699992656707764 - actor/pg_clipfrac:0.003355289101212596 - actor/ppo_kl:-1.924656582256526e-05 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.002379335288424045 - actor/loss:-0.002379337325692177 - actor/grad_norm:1.853515625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.337881088256836 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:117 - training/epoch:1 - critic/score/mean:0.5146484375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5146484375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.003735086414963007 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.003735086414963007 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:105.37890625 - response_length/max:128.0 - response_length/min:38.0 - response_length/clip_ratio:0.3720703125 - response_length_non_aborted/mean:105.37890625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:38.0 - response_length_non_aborted/clip_ratio:0.3720703125 - response/aborted_ratio:0.0 - prompt_length/mean:32.34375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:6.191547450005601 - timing_s/old_log_prob:7.941423087999283 - timing_s/adv:0.18186593300197273 - timing_s/update_actor:25.009928223000315 - timing_s/update_weights:1.9323736549995374 - timing_s/step:41.616864703995816 - timing_per_token_ms/adv:0.001289573226607289 - timing_per_token_ms/update_actor:0.17734016098221853 - timing_per_token_ms/gen:0.05737802062873559 - perf/total_num_tokens:141028 - perf/time_per_step:41.616864703995816 - perf/throughput:3388.722360588093 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:118 - global_seqlen/min:139859.0 - global_seqlen/max:139859.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:139859.0 - global_seqlen/balanced_max:139859.0 - global_seqlen/mean:139859.0 - actor/entropy:0.12942688167095184 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7479878664016724 - training/rollout_probs_diff_mean:0.004375600256025791 - training/rollout_probs_diff_std:0.015147021040320396 - training/rollout_actor_probs_pearson_corr:0.9965032935142517 - rollout_corr/training_ppl:1.14630925655365 - rollout_corr/training_log_ppl:0.1347319781780243 - rollout_corr/kl:0.0010980238439515233 - rollout_corr/k3_kl:0.0009120378526858985 - rollout_corr/rollout_ppl:1.1450260877609253 - rollout_corr/rollout_log_ppl:0.13362950086593628 - rollout_corr/log_ppl_diff:0.0011024605482816696 - rollout_corr/log_ppl_abs_diff:0.0035395417362451553 - rollout_corr/log_ppl_diff_max:0.03449779748916626 - rollout_corr/log_ppl_diff_min:-0.01655903458595276 - rollout_corr/ppl_ratio:1.0011138916015625 - rollout_corr/chi2_token:0.0013915300369262695 - rollout_corr/chi2_seq:0.14954876899719238 - actor/pg_clipfrac:0.0030352353520963033 - actor/ppo_kl:1.5644401325213636e-05 - actor/pg_clipfrac_lower:7.972955548514923e-06 - actor/pg_loss:0.0004408289969433099 - actor/loss:0.0004408283275552094 - actor/grad_norm:1.798828125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.32654094696045 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:118 - training/epoch:1 - critic/score/mean:0.4833984375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4833984375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0031076562590897083 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.0031076562590897083 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.5263671875 - response_length/max:128.0 - response_length/min:38.0 - response_length/clip_ratio:0.38671875 - response_length_non_aborted/mean:104.5263671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:38.0 - response_length_non_aborted/clip_ratio:0.38671875 - response/aborted_ratio:0.0 - prompt_length/mean:32.0546875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.18996002500353 - timing_s/old_log_prob:8.023511595994933 - timing_s/adv:0.19257219500286737 - timing_s/update_actor:24.862344054999994 - timing_s/update_weights:1.8875900159982848 - timing_s/step:43.32956646000093 - timing_per_token_ms/adv:0.0013769024160251923 - timing_per_token_ms/update_actor:0.17776720879600164 - timing_per_token_ms/gen:0.07651665366472209 - perf/total_num_tokens:139859 - perf/time_per_step:43.32956646000093 - perf/throughput:3227.7959699668086 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:119 - global_seqlen/min:139620.0 - global_seqlen/max:139620.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:139620.0 - global_seqlen/balanced_max:139620.0 - global_seqlen/mean:139620.0 - actor/entropy:0.1328607052564621 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9695179462432861 - training/rollout_probs_diff_mean:0.004569461569190025 - training/rollout_probs_diff_std:0.016113556921482086 - training/rollout_actor_probs_pearson_corr:0.9962742328643799 - rollout_corr/training_ppl:1.1555395126342773 - rollout_corr/training_log_ppl:0.14258210361003876 - rollout_corr/kl:0.0013576949713751674 - rollout_corr/k3_kl:0.001046417048200965 - rollout_corr/rollout_ppl:1.1538124084472656 - rollout_corr/rollout_log_ppl:0.14113685488700867 - rollout_corr/log_ppl_diff:0.0014452405739575624 - rollout_corr/log_ppl_abs_diff:0.0037720873951911926 - rollout_corr/log_ppl_diff_max:0.06421796977519989 - rollout_corr/log_ppl_diff_min:-0.015490800142288208 - rollout_corr/ppl_ratio:1.0014615058898926 - rollout_corr/chi2_token:0.0012221336364746094 - rollout_corr/chi2_seq:0.1700824499130249 - actor/pg_clipfrac:0.003205787463230081 - actor/ppo_kl:-0.00010680659685628294 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.00038540398236364126 - actor/loss:-0.00038540642708539963 - actor/grad_norm:1.884765625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.35700511932373 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:119 - training/epoch:1 - critic/score/mean:0.533203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.533203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.003129708580672741 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.003129708580672741 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.66796875 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.3671875 - response_length_non_aborted/mean:104.66796875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.3671875 - response/aborted_ratio:0.0 - prompt_length/mean:31.6796875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.189059811003972 - timing_s/old_log_prob:7.93721230099618 - timing_s/adv:0.18215502999373712 - timing_s/update_actor:24.798495105002075 - timing_s/update_weights:1.9151345529971877 - timing_s/step:43.3803023510045 - timing_per_token_ms/adv:0.0013046485460087175 - timing_per_token_ms/update_actor:0.177614203588326 - timing_per_token_ms/gen:0.07640473792688908 - perf/total_num_tokens:139620 - perf/time_per_step:43.3803023510045 - perf/throughput:3218.51145412238 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 86%|████████▌ | 120/140 [1:34:51<15:31, 46.55s/it] (TaskRunnerV1 pid=2351) Training Progress: 86%|████████▋ | 121/140 [1:35:35<14:28, 45.72s/it] (TaskRunnerV1 pid=2351) Training Progress: 87%|████████▋ | 122/140 [1:36:18<13:32, 45.13s/it] (TaskRunnerV1 pid=2351) step:120 - global_seqlen/min:140750.0 - global_seqlen/max:140750.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:140750.0 - global_seqlen/balanced_max:140750.0 - global_seqlen/mean:140750.0 - actor/entropy:0.13251955807209015 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9431430697441101 - training/rollout_probs_diff_mean:0.0045110853388905525 - training/rollout_probs_diff_std:0.016040604561567307 - training/rollout_actor_probs_pearson_corr:0.9963111877441406 - rollout_corr/training_ppl:1.1538022756576538 - rollout_corr/training_log_ppl:0.14098192751407623 - rollout_corr/kl:0.0008992644143290818 - rollout_corr/k3_kl:0.0010869532125070691 - rollout_corr/rollout_ppl:1.1527416706085205 - rollout_corr/rollout_log_ppl:0.14010827243328094 - rollout_corr/log_ppl_diff:0.0008736466988921165 - rollout_corr/log_ppl_abs_diff:0.0036296530161052942 - rollout_corr/log_ppl_diff_max:0.0694487988948822 - rollout_corr/log_ppl_diff_min:-0.016064748167991638 - rollout_corr/ppl_ratio:1.0008900165557861 - rollout_corr/chi2_token:0.002304553985595703 - rollout_corr/chi2_seq:0.22031700611114502 - actor/pg_clipfrac:0.0032348165599008403 - actor/ppo_kl:0.0002558107421843185 - actor/pg_clipfrac_lower:1.6722715978782315e-05 - actor/pg_loss:-0.00023474576300941408 - actor/loss:-0.0002347456756979227 - actor/grad_norm:2.587890625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.40394973754883 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.535) - val-core/char_count/acc/mean@1:np.float64(0.535) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:120 - training/epoch:1 - critic/score/mean:0.52734375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.52734375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0059595294296741486 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0059595294296741486 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:105.333984375 - response_length/max:128.0 - response_length/min:39.0 - response_length/clip_ratio:0.3837890625 - response_length_non_aborted/mean:105.333984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:39.0 - response_length_non_aborted/clip_ratio:0.3837890625 - response/aborted_ratio:0.0 - prompt_length/mean:32.1171875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.185793646000093 - timing_s/old_log_prob:7.939432150997163 - timing_s/adv:0.18056902299576905 - timing_s/update_actor:24.7750485630022 - timing_s/update_weights:1.926707364997128 - timing_s/step:43.19411528799537 - timing_s/testing:8.839107591003994 - timing_per_token_ms/adv:0.001282906024836725 - timing_per_token_ms/update_actor:0.1760216594174224 - timing_per_token_ms/gen:0.07589135790176423 - perf/total_num_tokens:140750 - perf/time_per_step:43.19411528799537 - perf/throughput:3258.5457315551876 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:121 - global_seqlen/min:141850.0 - global_seqlen/max:141850.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:141850.0 - global_seqlen/balanced_max:141850.0 - global_seqlen/mean:141850.0 - actor/entropy:0.13194379210472107 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9844352006912231 - training/rollout_probs_diff_mean:0.004540410824120045 - training/rollout_probs_diff_std:0.017101341858506203 - training/rollout_actor_probs_pearson_corr:0.9956679940223694 - rollout_corr/training_ppl:1.1553682088851929 - rollout_corr/training_log_ppl:0.14209090173244476 - rollout_corr/kl:0.0010915842140093446 - rollout_corr/k3_kl:0.001062400289811194 - rollout_corr/rollout_ppl:1.1540504693984985 - rollout_corr/rollout_log_ppl:0.14097067713737488 - rollout_corr/log_ppl_diff:0.0011202202877029777 - rollout_corr/log_ppl_abs_diff:0.0038194090593606234 - rollout_corr/log_ppl_diff_max:0.04413585364818573 - rollout_corr/log_ppl_diff_min:-0.017807409167289734 - rollout_corr/ppl_ratio:1.001134991645813 - rollout_corr/chi2_token:0.001784205436706543 - rollout_corr/chi2_seq:0.2344954013824463 - actor/pg_clipfrac:0.002791823135339655 - actor/ppo_kl:0.0007147621545300353 - actor/pg_clipfrac_lower:1.2580515431181993e-05 - actor/pg_loss:-0.0011244987981626764 - actor/loss:-0.0011244970373809338 - actor/grad_norm:2.0126953125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.353532791137695 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:121 - training/epoch:1 - critic/score/mean:0.447265625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.447265625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.010362607426941395 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.010362607426941395 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:105.666015625 - response_length/max:128.0 - response_length/min:29.0 - response_length/clip_ratio:0.4111328125 - response_length_non_aborted/mean:105.666015625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:29.0 - response_length_non_aborted/clip_ratio:0.4111328125 - response/aborted_ratio:0.0 - prompt_length/mean:32.859375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.193763883995416 - timing_s/old_log_prob:8.03606216499611 - timing_s/adv:0.18243361100030597 - timing_s/update_actor:25.093431150999095 - timing_s/update_weights:1.9076470459986012 - timing_s/step:43.580984275002265 - timing_per_token_ms/adv:0.0012861022982044834 - timing_per_token_ms/update_actor:0.17690117131476274 - timing_per_token_ms/gen:0.07572654742052287 - perf/total_num_tokens:141850 - perf/time_per_step:43.580984275002265 - perf/throughput:3254.8599431556236 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:122 - global_seqlen/min:141358.0 - global_seqlen/max:141358.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:141358.0 - global_seqlen/balanced_max:141358.0 - global_seqlen/mean:141358.0 - actor/entropy:0.12808263301849365 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8339666724205017 - training/rollout_probs_diff_mean:0.0043778810650110245 - training/rollout_probs_diff_std:0.015314876101911068 - training/rollout_actor_probs_pearson_corr:0.9964498281478882 - rollout_corr/training_ppl:1.148634910583496 - rollout_corr/training_log_ppl:0.13634948432445526 - rollout_corr/kl:0.0008995863026939332 - rollout_corr/k3_kl:0.0010383325861766934 - rollout_corr/rollout_ppl:1.1474952697753906 - rollout_corr/rollout_log_ppl:0.13538336753845215 - rollout_corr/log_ppl_diff:0.0009661327349022031 - rollout_corr/log_ppl_abs_diff:0.0036174156703054905 - rollout_corr/log_ppl_diff_max:0.05637684464454651 - rollout_corr/log_ppl_diff_min:-0.026512950658798218 - rollout_corr/ppl_ratio:1.0009808540344238 - rollout_corr/chi2_token:0.0038465261459350586 - rollout_corr/chi2_seq:0.3465474843978882 - actor/pg_clipfrac:0.0035636297398013994 - actor/ppo_kl:0.0003293915629001276 - actor/pg_clipfrac_lower:1.5141879581885103e-05 - actor/pg_loss:0.002214072912465781 - actor/loss:0.0022140720393508673 - actor/grad_norm:1.8486328125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.29272413253784 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:122 - training/epoch:1 - critic/score/mean:0.5166015625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5166015625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00990439672023058 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.00990439672023058 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:105.310546875 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.4033203125 - response_length_non_aborted/mean:105.310546875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.4033203125 - response/aborted_ratio:0.0 - prompt_length/mean:32.734375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.217196943995077 - timing_s/old_log_prob:7.9385050180062535 - timing_s/adv:0.18560718499793438 - timing_s/update_actor:25.02763514900289 - timing_s/update_weights:1.9060085600067396 - timing_s/step:43.445464496995555 - timing_per_token_ms/adv:0.0013130292236586143 - timing_per_token_ms/update_actor:0.17705142368315122 - timing_per_token_ms/gen:0.07619945607295274 - perf/total_num_tokens:141358 - perf/time_per_step:43.445464496995555 - perf/throughput:3253.6883110036842 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TransferQueueController pid=2438) 2026-11-01 18:52:34,059 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 32275, Total req/min: 6198.20, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.21 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=12035, req/min=2311.24, avg_time=0.000253s, max_time=0.073311s, min_time=0.000062s; CLEAR_META: req_count=16, req/min=3.07, avg_time=0.001789s, max_time=0.003800s, min_time=0.000843s; SET_CUSTOM_META: req_count=11989, req/min=2302.41, avg_time=0.000009s, max_time=0.001312s, min_time=0.000003s; KV_LIST: req_count=39, req/min=7.49, avg_time=0.000232s, max_time=0.000885s, min_time=0.000035s; NOTIFY_DATA_UPDATE: req_count=8189, req/min=1572.64, avg_time=0.000082s, max_time=0.000716s, min_time=0.000030s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.34, avg_time=0.000132s, max_time=0.000198s, min_time=0.000116s (TaskRunnerV1 pid=2351) Training Progress: 88%|████████▊ | 123/140 [1:37:02<12:40, 44.73s/it] (TaskRunnerV1 pid=2351) Training Progress: 89%|████████▊ | 124/140 [1:37:46<11:52, 44.52s/it] (SimpleStorageUnit pid=2596) 2026-11-01 18:54:04,952 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_72b4f821: [Performance] Total success requests: 1086, Total req/min: 204.96, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2596) Time range: last 5.30 minutes; (SimpleStorageUnit pid=2596) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.02, avg_time=0.001512s, max_time=0.004150s, min_time=0.000420s; PUT_DATA: req_count=1034, req/min=195.14, avg_time=0.000623s, max_time=0.001831s, min_time=0.000320s; GET_DATA: req_count=36, req/min=6.79, avg_time=0.000634s, max_time=0.001234s, min_time=0.000416s (TaskRunnerV1 pid=2351) Training Progress: 89%|████████▉ | 125/140 [1:38:40<11:46, 47.12s/it] (SimpleStorageUnit pid=2583) 2026-11-01 18:54:05,841 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1099, Total req/min: 206.83, Total avg process time: 0.0006s; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.31 minutes; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.01, avg_time=0.001443s, max_time=0.003874s, min_time=0.000416s; PUT_DATA: req_count=1047, req/min=197.05, avg_time=0.000600s, max_time=0.001597s, min_time=0.000316s; GET_DATA: req_count=36, req/min=6.78, avg_time=0.000640s, max_time=0.001275s, min_time=0.000400s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) step:123 - global_seqlen/min:142824.0 - global_seqlen/max:142824.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:142824.0 - global_seqlen/balanced_max:142824.0 - global_seqlen/mean:142824.0 - actor/entropy:0.12820814549922943 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9389788508415222 - training/rollout_probs_diff_mean:0.0044569410383701324 - training/rollout_probs_diff_std:0.01758885756134987 - training/rollout_actor_probs_pearson_corr:0.9952754974365234 - rollout_corr/training_ppl:1.149268627166748 - rollout_corr/training_log_ppl:0.13723452389240265 - rollout_corr/kl:0.0009770566830411553 - rollout_corr/k3_kl:0.0010790335945785046 - rollout_corr/rollout_ppl:1.1481902599334717 - rollout_corr/rollout_log_ppl:0.13631199300289154 - rollout_corr/log_ppl_diff:0.0009225307730957866 - rollout_corr/log_ppl_abs_diff:0.0035705368500202894 - rollout_corr/log_ppl_diff_max:0.027462869882583618 - rollout_corr/log_ppl_diff_min:-0.015949353575706482 - rollout_corr/ppl_ratio:1.0009353160858154 - rollout_corr/chi2_token:0.0021284818649291992 - rollout_corr/chi2_seq:0.2433995008468628 - actor/pg_clipfrac:0.003295890463050455 - actor/ppo_kl:4.2000980935578524e-05 - actor/pg_clipfrac_lower:4.602748837593632e-05 - actor/pg_loss:-7.59208487579599e-05 - actor/loss:-7.592386100441217e-05 - actor/grad_norm:1.8134765625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.28183126449585 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:123 - training/epoch:1 - critic/score/mean:0.490234375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.490234375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.00554957240819931 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.00554957240819931 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:106.84375 - response_length/max:128.0 - response_length/min:38.0 - response_length/clip_ratio:0.4287109375 - response_length_non_aborted/mean:106.84375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:38.0 - response_length_non_aborted/clip_ratio:0.4287109375 - response/aborted_ratio:0.0 - prompt_length/mean:32.6328125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.18975420499919 - timing_s/old_log_prob:8.009813734999625 - timing_s/adv:0.1833522580054705 - timing_s/update_actor:25.19857160700485 - timing_s/update_weights:1.931299528994714 - timing_s/step:43.68066087500483 - timing_per_token_ms/adv:0.0012837636392025885 - timing_per_token_ms/update_actor:0.1764309332255423 - timing_per_token_ms/gen:0.07485516785791889 - perf/total_num_tokens:142824 - perf/time_per_step:43.68066087500483 - perf/throughput:3269.7307490081375 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:124 - global_seqlen/min:143581.0 - global_seqlen/max:143581.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:143581.0 - global_seqlen/balanced_max:143581.0 - global_seqlen/mean:143581.0 - actor/entropy:0.1286001205444336 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.756602942943573 - training/rollout_probs_diff_mean:0.004430695436894894 - training/rollout_probs_diff_std:0.01552000641822815 - training/rollout_actor_probs_pearson_corr:0.9962462186813354 - rollout_corr/training_ppl:1.1465550661087036 - rollout_corr/training_log_ppl:0.13459941744804382 - rollout_corr/kl:0.0005769636482000351 - rollout_corr/k3_kl:0.0009834493976086378 - rollout_corr/rollout_ppl:1.1458739042282104 - rollout_corr/rollout_log_ppl:0.1340257227420807 - rollout_corr/log_ppl_diff:0.0005736899911426008 - rollout_corr/log_ppl_abs_diff:0.003531716763973236 - rollout_corr/log_ppl_diff_max:0.0350477397441864 - rollout_corr/log_ppl_diff_min:-0.01741349697113037 - rollout_corr/ppl_ratio:1.0005850791931152 - rollout_corr/chi2_token:0.0030105113983154297 - rollout_corr/chi2_seq:0.4238553047180176 - actor/pg_clipfrac:0.002456877560083133 - actor/ppo_kl:0.000255087903800207 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:-0.0008117289398796856 - actor/loss:-0.0008117295801639557 - actor/grad_norm:1.8701171875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.3634352684021 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:124 - training/epoch:1 - critic/score/mean:0.515625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.515625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00021008097974117845 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.00021008097974117845 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.8173828125 - response_length/max:128.0 - response_length/min:36.0 - response_length/clip_ratio:0.40625 - response_length_non_aborted/mean:107.8173828125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:36.0 - response_length_non_aborted/clip_ratio:0.40625 - response/aborted_ratio:0.0 - prompt_length/mean:32.3984375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.193523103996995 - timing_s/old_log_prob:8.215879945004417 - timing_s/adv:0.18461146999470657 - timing_s/update_actor:25.2439762110007 - timing_s/update_weights:1.9066511260025436 - timing_s/step:43.910475960998156 - timing_per_token_ms/adv:0.0012857653171011942 - timing_per_token_ms/update_actor:0.17581696889561083 - timing_per_token_ms/gen:0.07421333367145505 - perf/total_num_tokens:143581 - perf/time_per_step:43.910475960998156 - perf/throughput:3269.8575193657766 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:125 - global_seqlen/min:144380.0 - global_seqlen/max:144380.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:144380.0 - global_seqlen/balanced_max:144380.0 - global_seqlen/mean:144380.0 - actor/entropy:0.12894131243228912 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9385102987289429 - training/rollout_probs_diff_mean:0.00434386869892478 - training/rollout_probs_diff_std:0.015413956716656685 - training/rollout_actor_probs_pearson_corr:0.9962508082389832 - rollout_corr/training_ppl:1.1468250751495361 - rollout_corr/training_log_ppl:0.13501733541488647 - rollout_corr/kl:0.000974722788669169 - rollout_corr/k3_kl:0.000987500767223537 - rollout_corr/rollout_ppl:1.1456958055496216 - rollout_corr/rollout_log_ppl:0.1340589076280594 - rollout_corr/log_ppl_diff:0.0009584141662344337 - rollout_corr/log_ppl_abs_diff:0.0033311599399894476 - rollout_corr/log_ppl_diff_max:0.024919122457504272 - rollout_corr/log_ppl_diff_min:-0.02633938193321228 - rollout_corr/ppl_ratio:1.0009688138961792 - rollout_corr/chi2_token:0.0038825273513793945 - rollout_corr/chi2_seq:0.9656980037689209 - actor/pg_clipfrac:0.002963447036260429 - actor/ppo_kl:0.00011663800963409207 - actor/pg_clipfrac_lower:2.3806197229229532e-05 - actor/pg_loss:-0.001177442172775045 - actor/loss:-0.0011774416198022664 - actor/grad_norm:1.779296875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.41069173812866 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.537) - val-core/char_count/acc/mean@1:np.float64(0.537) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:125 - training/epoch:1 - critic/score/mean:0.4658203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4658203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.00048457630327902734 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.00048457630327902734 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:108.18359375 - response_length/max:128.0 - response_length/min:27.0 - response_length/clip_ratio:0.4443359375 - response_length_non_aborted/mean:108.18359375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:27.0 - response_length_non_aborted/clip_ratio:0.4443359375 - response/aborted_ratio:0.0 - prompt_length/mean:32.8125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.191839835999417 - timing_s/old_log_prob:8.207641592998698 - timing_s/adv:0.18427295300352853 - timing_s/update_actor:25.464820542998496 - timing_s/update_weights:1.8912416340026539 - timing_s/step:44.110394798000925 - timing_s/testing:8.725892889000534 - timing_per_token_ms/adv:0.001276305256985237 - timing_per_token_ms/update_actor:0.17637360121206883 - timing_per_token_ms/gen:0.0739469203466277 - perf/total_num_tokens:144380 - perf/time_per_step:44.110394798000925 - perf/throughput:3273.1513889452485 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 90%|█████████ | 126/140 [1:39:23<10:43, 45.99s/it] (TaskRunnerV1 pid=2351) Training Progress: 91%|█████████ | 127/140 [1:40:04<09:37, 44.45s/it] (TaskRunnerV1 pid=2351) Training Progress: 91%|█████████▏| 128/140 [1:40:45<08:41, 43.50s/it] (TaskRunnerV1 pid=2351) step:126 - global_seqlen/min:140788.0 - global_seqlen/max:140788.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:140788.0 - global_seqlen/balanced_max:140788.0 - global_seqlen/mean:140788.0 - actor/entropy:0.13168810307979584 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6936917901039124 - training/rollout_probs_diff_mean:0.0044647445902228355 - training/rollout_probs_diff_std:0.015429123304784298 - training/rollout_actor_probs_pearson_corr:0.9962980151176453 - rollout_corr/training_ppl:1.149747610092163 - rollout_corr/training_log_ppl:0.1375449150800705 - rollout_corr/kl:0.0011700253235176206 - rollout_corr/k3_kl:0.0010702054714784026 - rollout_corr/rollout_ppl:1.148418664932251 - rollout_corr/rollout_log_ppl:0.13640622794628143 - rollout_corr/log_ppl_diff:0.0011386852711439133 - rollout_corr/log_ppl_abs_diff:0.003539382480084896 - rollout_corr/log_ppl_diff_max:0.05017653852701187 - rollout_corr/log_ppl_diff_min:-0.01932618021965027 - rollout_corr/ppl_ratio:1.0011528730392456 - rollout_corr/chi2_token:0.001941084861755371 - rollout_corr/chi2_seq:0.26817119121551514 - actor/pg_clipfrac:0.0033521684138880423 - actor/ppo_kl:-0.0001384151485884407 - actor/pg_clipfrac_lower:1.5141879581885103e-05 - actor/pg_loss:0.0007257345860125497 - actor/loss:0.0007257340475916862 - actor/grad_norm:2.2392578125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.37001657485962 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:126 - training/epoch:1 - critic/score/mean:0.517578125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.517578125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.002143259858712554 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.002143259858712554 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.86328125 - response_length/max:128.0 - response_length/min:43.0 - response_length/clip_ratio:0.3701171875 - response_length_non_aborted/mean:104.86328125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:43.0 - response_length_non_aborted/clip_ratio:0.3701171875 - response/aborted_ratio:0.0 - prompt_length/mean:32.625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:8.190121425999678 - timing_s/old_log_prob:7.928898304999166 - timing_s/adv:0.18371232299978146 - timing_s/update_actor:24.79026102899661 - timing_s/update_weights:1.9424909750014194 - timing_s/step:43.203964819993416 - timing_per_token_ms/adv:0.0013048862332001412 - timing_per_token_ms/update_actor:0.176082201814051 - timing_per_token_ms/gen:0.07627231724715662 - perf/total_num_tokens:140788 - perf/time_per_step:43.203964819993416 - perf/throughput:3258.682405343683 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:127 - global_seqlen/min:135765.0 - global_seqlen/max:135765.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:135765.0 - global_seqlen/balanced_max:135765.0 - global_seqlen/mean:135765.0 - actor/entropy:0.13397589325904846 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6357094049453735 - training/rollout_probs_diff_mean:0.004613654222339392 - training/rollout_probs_diff_std:0.015322172082960606 - training/rollout_actor_probs_pearson_corr:0.9965068697929382 - rollout_corr/training_ppl:1.1539523601531982 - rollout_corr/training_log_ppl:0.1411767154932022 - rollout_corr/kl:0.000879951287060976 - rollout_corr/k3_kl:0.0009781979024410248 - rollout_corr/rollout_ppl:1.152809739112854 - rollout_corr/rollout_log_ppl:0.14023347198963165 - rollout_corr/log_ppl_diff:0.0009432443184778094 - rollout_corr/log_ppl_abs_diff:0.003700768342241645 - rollout_corr/log_ppl_diff_max:0.050722360610961914 - rollout_corr/log_ppl_diff_min:-0.0257740318775177 - rollout_corr/ppl_ratio:1.0009567737579346 - rollout_corr/chi2_token:0.002634286880493164 - rollout_corr/chi2_seq:0.21409153938293457 - actor/pg_clipfrac:0.003315069683594629 - actor/ppo_kl:0.00015602073024941396 - actor/pg_clipfrac_lower:0.0 - actor/pg_loss:0.0003624691744334996 - actor/loss:0.0003624693490564823 - actor/grad_norm:2.09375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.2897310256958 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:127 - training/epoch:1 - critic/score/mean:0.533203125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.533203125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0031073640566319227 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0031073640566319227 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:100.5361328125 - response_length/max:128.0 - response_length/min:29.0 - response_length/clip_ratio:0.3251953125 - response_length_non_aborted/mean:100.5361328125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:29.0 - response_length_non_aborted/clip_ratio:0.3251953125 - response/aborted_ratio:0.0 - prompt_length/mean:32.046875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:6.201012011006242 - timing_s/old_log_prob:7.690303297000355 - timing_s/adv:0.1881820139969932 - timing_s/update_actor:24.365053044995875 - timing_s/update_weights:1.9382565039995825 - timing_s/step:40.738661466995836 - timing_per_token_ms/adv:0.001386086355076737 - timing_per_token_ms/update_actor:0.17946490660329154 - timing_per_token_ms/gen:0.060233824621960794 - perf/total_num_tokens:135765 - perf/time_per_step:40.738661466995836 - perf/throughput:3332.583720503167 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:128 - global_seqlen/min:137850.0 - global_seqlen/max:137850.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:137850.0 - global_seqlen/balanced_max:137850.0 - global_seqlen/mean:137850.0 - actor/entropy:0.12867580354213715 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8434796929359436 - training/rollout_probs_diff_mean:0.0043962262570858 - training/rollout_probs_diff_std:0.015813620761036873 - training/rollout_actor_probs_pearson_corr:0.9962652325630188 - rollout_corr/training_ppl:1.1507536172866821 - rollout_corr/training_log_ppl:0.13813281059265137 - rollout_corr/kl:0.0010396059369668365 - rollout_corr/k3_kl:0.0009195713791996241 - rollout_corr/rollout_ppl:1.1495380401611328 - rollout_corr/rollout_log_ppl:0.1370987743139267 - rollout_corr/log_ppl_diff:0.001034051296301186 - rollout_corr/log_ppl_abs_diff:0.003511809278279543 - rollout_corr/log_ppl_diff_max:0.031620755791664124 - rollout_corr/log_ppl_diff_min:-0.033657580614089966 - rollout_corr/ppl_ratio:1.0010464191436768 - rollout_corr/chi2_token:0.0018041133880615234 - rollout_corr/chi2_seq:0.2154865264892578 - actor/pg_clipfrac:0.0032716043206164613 - actor/ppo_kl:-0.00018561918614068418 - actor/pg_clipfrac_lower:2.4590948669356294e-05 - actor/pg_loss:0.0016162741885636933 - actor/loss:0.0016162734536919743 - actor/grad_norm:1.703125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.28040361404419 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:128 - training/epoch:1 - critic/score/mean:0.484375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.484375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0011707531521096826 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0011707531521096826 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:102.439453125 - response_length/max:128.0 - response_length/min:39.0 - response_length/clip_ratio:0.341796875 - response_length_non_aborted/mean:102.439453125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:39.0 - response_length_non_aborted/clip_ratio:0.341796875 - response/aborted_ratio:0.0 - prompt_length/mean:32.1796875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:6.217145773000084 - timing_s/old_log_prob:7.937218065002526 - timing_s/adv:0.18207462099962868 - timing_s/update_actor:24.70344577299693 - timing_s/update_weights:1.92372993400204 - timing_s/step:41.132853745999455 - timing_per_token_ms/adv:0.0013208169822243648 - timing_per_token_ms/update_actor:0.1792052649473843 - timing_per_token_ms/gen:0.05926848722568671 - perf/total_num_tokens:137850 - perf/time_per_step:41.132853745999455 - perf/throughput:3351.335670781344 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 92%|█████████▏| 129/140 [1:41:28<07:58, 43.46s/it] (TransferQueueController pid=2438) 2026-11-01 18:57:43,694 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 32274, Total req/min: 6253.94, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.16 minutes; (TransferQueueController pid=2438) Per-operation statistics: KV_RETRIEVE_META: req_count=12035, req/min=2332.10, avg_time=0.000256s, max_time=0.067732s, min_time=0.000063s; CLEAR_META: req_count=16, req/min=3.10, avg_time=0.001985s, max_time=0.003999s, min_time=0.000932s; SET_CUSTOM_META: req_count=11989, req/min=2323.19, avg_time=0.000009s, max_time=0.000225s, min_time=0.000003s; KV_LIST: req_count=38, req/min=7.36, avg_time=0.000237s, max_time=0.000921s, min_time=0.000056s; NOTIFY_DATA_UPDATE: req_count=8189, req/min=1586.84, avg_time=0.000082s, max_time=0.000681s, min_time=0.000031s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.36, avg_time=0.000167s, max_time=0.000240s, min_time=0.000107s (TaskRunnerV1 pid=2351) Training Progress: 93%|█████████▎| 130/140 [1:42:20<07:40, 46.00s/it] (TaskRunnerV1 pid=2351) Training Progress: 94%|█████████▎| 131/140 [1:43:04<06:47, 45.30s/it] (TaskRunnerV1 pid=2351) step:129 - global_seqlen/min:139527.0 - global_seqlen/max:139527.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:139527.0 - global_seqlen/balanced_max:139527.0 - global_seqlen/mean:139527.0 - actor/entropy:0.13589654862880707 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7767482995986938 - training/rollout_probs_diff_mean:0.0046819234266877174 - training/rollout_probs_diff_std:0.015536867082118988 - training/rollout_actor_probs_pearson_corr:0.9963935017585754 - rollout_corr/training_ppl:1.1573892831802368 - rollout_corr/training_log_ppl:0.14399036765098572 - rollout_corr/kl:0.0010220201220363379 - rollout_corr/k3_kl:0.0010261526331305504 - rollout_corr/rollout_ppl:1.1561734676361084 - rollout_corr/rollout_log_ppl:0.14296993613243103 - rollout_corr/log_ppl_diff:0.0010204346617683768 - rollout_corr/log_ppl_abs_diff:0.0036716454196721315 - rollout_corr/log_ppl_diff_max:0.07045277208089828 - rollout_corr/log_ppl_diff_min:-0.01876874268054962 - rollout_corr/ppl_ratio:1.0010349750518799 - rollout_corr/chi2_token:0.001934647560119629 - rollout_corr/chi2_seq:0.22967970371246338 - actor/pg_clipfrac:0.0031214445722677437 - actor/ppo_kl:-4.500747820657125e-05 - actor/pg_clipfrac_lower:1.5353957375433918e-05 - actor/pg_loss:0.0007938389971968718 - actor/loss:0.0007938393391668797 - actor/grad_norm:1.7783203125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.37061262130737 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:129 - training/epoch:1 - critic/score/mean:0.5498046875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5498046875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.001619677059352398 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.001619677059352398 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.1630859375 - response_length/max:128.0 - response_length/min:33.0 - response_length/clip_ratio:0.373046875 - response_length_non_aborted/mean:104.1630859375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:33.0 - response_length_non_aborted/clip_ratio:0.373046875 - response/aborted_ratio:0.0 - prompt_length/mean:32.09375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0859375 - timing_s/gen:8.187447330994473 - timing_s/old_log_prob:8.113771702999657 - timing_s/adv:0.18347914399782894 - timing_s/update_actor:24.64474262300064 - timing_s/update_weights:1.897578345000511 - timing_s/step:43.19796567200683 - timing_per_token_ms/adv:0.0013150081632789994 - timing_per_token_ms/update_actor:0.17663063509572083 - timing_per_token_ms/gen:0.07675995735160715 - perf/total_num_tokens:139527 - perf/time_per_step:43.19796567200683 - perf/throughput:3229.943767708866 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:130 - global_seqlen/min:138152.0 - global_seqlen/max:138152.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:138152.0 - global_seqlen/balanced_max:138152.0 - global_seqlen/mean:138152.0 - actor/entropy:0.13426029682159424 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7763886451721191 - training/rollout_probs_diff_mean:0.00455118902027607 - training/rollout_probs_diff_std:0.016023727133870125 - training/rollout_actor_probs_pearson_corr:0.9961537718772888 - rollout_corr/training_ppl:1.1571346521377563 - rollout_corr/training_log_ppl:0.14359517395496368 - rollout_corr/kl:0.0009003951563499868 - rollout_corr/k3_kl:0.0009853766532614827 - rollout_corr/rollout_ppl:1.1560019254684448 - rollout_corr/rollout_log_ppl:0.14266428351402283 - rollout_corr/log_ppl_diff:0.0009309102897532284 - rollout_corr/log_ppl_abs_diff:0.003623498370870948 - rollout_corr/log_ppl_diff_max:0.04940196871757507 - rollout_corr/log_ppl_diff_min:-0.02519315481185913 - rollout_corr/ppl_ratio:1.000943899154663 - rollout_corr/chi2_token:0.002039790153503418 - rollout_corr/chi2_seq:0.25504612922668457 - actor/pg_clipfrac:0.0033908716238026195 - actor/ppo_kl:-0.0003004108233047494 - actor/pg_clipfrac_lower:1.7721182909250878e-05 - actor/pg_loss:8.521665586158633e-05 - actor/loss:8.521601557731628e-05 - actor/grad_norm:2.205078125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.297046184539795 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.537) - val-core/char_count/acc/mean@1:np.float64(0.537) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:130 - training/epoch:1 - critic/score/mean:0.51953125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.51953125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.002005802933126688 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.002005802933126688 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:102.8515625 - response_length/max:128.0 - response_length/min:30.0 - response_length/clip_ratio:0.3544921875 - response_length_non_aborted/mean:102.8515625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:30.0 - response_length_non_aborted/clip_ratio:0.3544921875 - response/aborted_ratio:0.0 - prompt_length/mean:32.0625 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.190419211001426 - timing_s/old_log_prob:7.73621260800428 - timing_s/adv:0.4673720080027124 - timing_s/update_actor:24.731668218999403 - timing_s/update_weights:1.8987453950030613 - timing_s/step:43.19269939700462 - timing_s/testing:8.626707644005364 - timing_per_token_ms/adv:0.003383027448047892 - timing_per_token_ms/update_actor:0.17901780805923478 - timing_per_token_ms/gen:0.07776698833081491 - perf/total_num_tokens:138152 - perf/time_per_step:43.19269939700462 - perf/throughput:3198.503495467587 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:131 - global_seqlen/min:140630.0 - global_seqlen/max:140630.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:140630.0 - global_seqlen/balanced_max:140630.0 - global_seqlen/mean:140630.0 - actor/entropy:0.12673510611057281 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.7980758547782898 - training/rollout_probs_diff_mean:0.00434883451089263 - training/rollout_probs_diff_std:0.015619899146258831 - training/rollout_actor_probs_pearson_corr:0.9961357116699219 - rollout_corr/training_ppl:1.1464667320251465 - rollout_corr/training_log_ppl:0.13462397456169128 - rollout_corr/kl:0.0009585609077475965 - rollout_corr/k3_kl:0.0010323073947802186 - rollout_corr/rollout_ppl:1.145363211631775 - rollout_corr/rollout_log_ppl:0.13368014991283417 - rollout_corr/log_ppl_diff:0.0009438404231332242 - rollout_corr/log_ppl_abs_diff:0.0035358008462935686 - rollout_corr/log_ppl_diff_max:0.06749048829078674 - rollout_corr/log_ppl_diff_min:-0.020285680890083313 - rollout_corr/ppl_ratio:1.00095796585083 - rollout_corr/chi2_token:0.002557039260864258 - rollout_corr/chi2_seq:0.3476431369781494 - actor/pg_clipfrac:0.0030713570934797945 - actor/ppo_kl:-3.9079751938212796e-06 - actor/pg_clipfrac_lower:9.056002454599366e-06 - actor/pg_loss:0.0006271344173001125 - actor/loss:0.0006271344609558582 - actor/grad_norm:1.890625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.39612817764282 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:131 - training/epoch:1 - critic/score/mean:0.490234375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.490234375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.001977452076971531 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.001977452076971531 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.521484375 - response_length/max:128.0 - response_length/min:34.0 - response_length/clip_ratio:0.4013671875 - response_length_non_aborted/mean:104.521484375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:34.0 - response_length_non_aborted/clip_ratio:0.4013671875 - response/aborted_ratio:0.0 - prompt_length/mean:32.8125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.078125 - timing_s/gen:8.200338561997341 - timing_s/old_log_prob:8.073305470999912 - timing_s/adv:0.2253107610013103 - timing_s/update_actor:24.939202618006675 - timing_s/update_weights:1.907884461004869 - timing_s/step:43.52202397899964 - timing_per_token_ms/adv:0.0016021528905732084 - timing_per_token_ms/update_actor:0.177339135447676 - timing_per_token_ms/gen:0.07661719669249127 - perf/total_num_tokens:140630 - perf/time_per_step:43.52202397899964 - perf/throughput:3231.237592899107 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(SimpleStorageUnit pid=2599) 2026-11-01 18:59:19,202 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1219, Total req/min: 233.08, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.23 minutes; (SimpleStorageUnit pid=2599) Per-operation statistics: PUT_DATA: req_count=1161, req/min=221.99, avg_time=0.000588s, max_time=0.002102s, min_time=0.000310s; CLEAR_DATA: req_count=18, req/min=3.44, avg_time=0.001347s, max_time=0.003626s, min_time=0.000389s; GET_DATA: req_count=40, req/min=7.65, avg_time=0.000567s, max_time=0.000889s, min_time=0.000367s (TaskRunnerV1 pid=2351) Training Progress: 94%|█████████▍| 132/140 [1:43:47<05:57, 44.75s/it] (TaskRunnerV1 pid=2351) Training Progress: 95%|█████████▌| 133/140 [1:44:31<05:10, 44.40s/it] (SimpleStorageUnit pid=2583) 2026-11-01 18:59:19,200 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1227, Total req/min: 234.94, Total avg process time: 0.0006s; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.22 minutes; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: PUT_DATA: req_count=1169, req/min=223.83, avg_time=0.000601s, max_time=0.001199s, min_time=0.000314s; CLEAR_DATA: req_count=18, req/min=3.45, avg_time=0.001694s, max_time=0.003916s, min_time=0.000392s; GET_DATA: req_count=40, req/min=7.66, avg_time=0.000637s, max_time=0.001210s, min_time=0.000404s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 96%|█████████▌| 134/140 [1:45:15<04:25, 44.33s/it] (TaskRunnerV1 pid=2351) step:132 - global_seqlen/min:139505.0 - global_seqlen/max:139505.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:139505.0 - global_seqlen/balanced_max:139505.0 - global_seqlen/mean:139505.0 - actor/entropy:0.12866388261318207 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9883034229278564 - training/rollout_probs_diff_mean:0.004511989653110504 - training/rollout_probs_diff_std:0.017618495970964432 - training/rollout_actor_probs_pearson_corr:0.9955093264579773 - rollout_corr/training_ppl:1.148900032043457 - rollout_corr/training_log_ppl:0.1370406299829483 - rollout_corr/kl:0.0014512885827571154 - rollout_corr/k3_kl:0.0013478246983140707 - rollout_corr/rollout_ppl:1.1470898389816284 - rollout_corr/rollout_log_ppl:0.13550414144992828 - rollout_corr/log_ppl_diff:0.0015364717692136765 - rollout_corr/log_ppl_abs_diff:0.004070669412612915 - rollout_corr/log_ppl_diff_max:0.18153345584869385 - rollout_corr/log_ppl_diff_min:-0.018032550811767578 - rollout_corr/ppl_ratio:1.0015743970870972 - rollout_corr/chi2_token:0.0018274784088134766 - rollout_corr/chi2_seq:0.22018003463745117 - actor/pg_clipfrac:0.003496028002700769 - actor/ppo_kl:-0.0003316358468813026 - actor/pg_clipfrac_lower:5.3849741561862174e-05 - actor/pg_loss:0.0006757214941899292 - actor/loss:0.0006757212104275823 - actor/grad_norm:2.3447265625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.41123294830322 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:132 - training/epoch:1 - critic/score/mean:0.5146484375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5146484375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.01044780109077692 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.01044780109077692 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.3056640625 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.3818359375 - response_length_non_aborted/mean:104.3056640625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.3818359375 - response/aborted_ratio:0.0 - prompt_length/mean:31.9296875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0234375 - timing_s/gen:8.202378387999488 - timing_s/old_log_prob:8.158021173003362 - timing_s/adv:0.18335727199882967 - timing_s/update_actor:24.725597476004623 - timing_s/update_weights:1.8988891109984252 - timing_s/step:43.34028200899775 - timing_per_token_ms/adv:0.0013143419375565726 - timing_per_token_ms/update_actor:0.17723807373215744 - timing_per_token_ms/gen:0.07679482429382813 - perf/total_num_tokens:139505 - perf/time_per_step:43.34028200899775 - perf/throughput:3218.830001406954 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:133 - global_seqlen/min:140227.0 - global_seqlen/max:140227.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:140227.0 - global_seqlen/balanced_max:140227.0 - global_seqlen/mean:140227.0 - actor/entropy:0.12549801170825958 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.938686728477478 - training/rollout_probs_diff_mean:0.004292341414839029 - training/rollout_probs_diff_std:0.015461845323443413 - training/rollout_actor_probs_pearson_corr:0.9963696599006653 - rollout_corr/training_ppl:1.145195484161377 - rollout_corr/training_log_ppl:0.13358846306800842 - rollout_corr/kl:0.0010769543005153537 - rollout_corr/k3_kl:0.0009027862688526511 - rollout_corr/rollout_ppl:1.143855333328247 - rollout_corr/rollout_log_ppl:0.13246648013591766 - rollout_corr/log_ppl_diff:0.0011219913139939308 - rollout_corr/log_ppl_abs_diff:0.003549718763679266 - rollout_corr/log_ppl_diff_max:0.056037068367004395 - rollout_corr/log_ppl_diff_min:-0.01414334774017334 - rollout_corr/ppl_ratio:1.0011346340179443 - rollout_corr/chi2_token:0.0013840198516845703 - rollout_corr/chi2_seq:0.17699813842773438 - actor/pg_clipfrac:0.003216156202446048 - actor/ppo_kl:0.0002636319932207698 - actor/pg_clipfrac_lower:1.058245834428817e-05 - actor/pg_loss:0.0014977495302446187 - actor/loss:0.0014977483951952308 - actor/grad_norm:2.1259765625 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.32665157318115 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:133 - training/epoch:1 - critic/score/mean:0.54296875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.54296875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008052260614931583 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008052260614931583 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.5810546875 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.35546875 - response_length_non_aborted/mean:104.5810546875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.35546875 - response/aborted_ratio:0.0 - prompt_length/mean:32.359375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:8.188033987004019 - timing_s/old_log_prob:8.131430669003748 - timing_s/adv:0.18278851299692178 - timing_s/update_actor:24.860751632993924 - timing_s/update_weights:1.9011386190031772 - timing_s/step:43.43506568700104 - timing_per_token_ms/adv:0.0013035186732720644 - timing_per_token_ms/update_actor:0.17728933538472566 - timing_per_token_ms/gen:0.07645865653513385 - perf/total_num_tokens:140227 - perf/time_per_step:43.43506568700104 - perf/throughput:3228.4284087537644 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:134 - global_seqlen/min:143701.0 - global_seqlen/max:143701.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:143701.0 - global_seqlen/balanced_max:143701.0 - global_seqlen/mean:143701.0 - actor/entropy:0.12563206255435944 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8816324472427368 - training/rollout_probs_diff_mean:0.004289884120225906 - training/rollout_probs_diff_std:0.015551183372735977 - training/rollout_actor_probs_pearson_corr:0.9962818622589111 - rollout_corr/training_ppl:1.1464595794677734 - rollout_corr/training_log_ppl:0.13467669486999512 - rollout_corr/kl:0.0008344108937308192 - rollout_corr/k3_kl:0.0009659765637479722 - rollout_corr/rollout_ppl:1.1454968452453613 - rollout_corr/rollout_log_ppl:0.13386332988739014 - rollout_corr/log_ppl_diff:0.0008133628871291876 - rollout_corr/log_ppl_abs_diff:0.003408326767385006 - rollout_corr/log_ppl_diff_max:0.02317708730697632 - rollout_corr/log_ppl_diff_min:-0.02308233082294464 - rollout_corr/ppl_ratio:1.000824213027954 - rollout_corr/chi2_token:0.00280153751373291 - rollout_corr/chi2_seq:0.5872769355773926 - actor/pg_clipfrac:0.0029534913919633254 - actor/ppo_kl:0.00020930100449353026 - actor/pg_clipfrac_lower:1.5621632580102112e-05 - actor/pg_loss:0.0010585013551462907 - actor/loss:0.001058500085491687 - actor/grad_norm:1.9033203125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.31385946273804 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:134 - training/epoch:1 - critic/score/mean:0.4814453125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4814453125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.008494354784488678 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.008494354784488678 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:107.3095703125 - response_length/max:128.0 - response_length/min:26.0 - response_length/clip_ratio:0.392578125 - response_length_non_aborted/mean:107.3095703125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:26.0 - response_length_non_aborted/clip_ratio:0.392578125 - response/aborted_ratio:0.0 - prompt_length/mean:33.0234375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:8.19593408700166 - timing_s/old_log_prob:8.004591516997607 - timing_s/adv:0.39333776100102114 - timing_s/update_actor:25.34830298399902 - timing_s/update_weights:1.9318289950024337 - timing_s/step:44.04770675900363 - timing_per_token_ms/adv:0.002737195711936738 - timing_per_token_ms/update_actor:0.17639614883681407 - timing_per_token_ms/gen:0.07458646846249861 - perf/total_num_tokens:143701 - perf/time_per_step:44.04770675900363 - perf/throughput:3262.394584722089 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 96%|█████████▋| 135/140 [1:46:08<03:53, 46.74s/it] (TaskRunnerV1 pid=2351) Training Progress: 97%|█████████▋| 136/140 [1:46:49<03:00, 45.16s/it] (TransferQueueController pid=2438) 2026-11-01 19:03:02,097 - INFO - transfer_queue.utils.perf_utils - TQ_CONTROLLER_77a4f4f5: [Performance] Total success requests: 39287, Total req/min: 7403.27, Total avg process time: 0.0001s; (TransferQueueController pid=2438) Time range: last 5.31 minutes; (TransferQueueController pid=2438) Per-operation statistics: SET_CUSTOM_META: req_count=14990, req/min=2824.72, avg_time=0.000009s, max_time=0.000280s, min_time=0.000003s; KV_RETRIEVE_META: req_count=15040, req/min=2834.15, avg_time=0.000238s, max_time=0.067743s, min_time=0.000063s; KV_LIST: req_count=43, req/min=8.10, avg_time=0.000296s, max_time=0.000969s, min_time=0.000051s; NOTIFY_DATA_UPDATE: req_count=9189, req/min=1731.58, avg_time=0.000082s, max_time=0.000741s, min_time=0.000030s; CLEAR_META: req_count=18, req/min=3.39, avg_time=0.001773s, max_time=0.003759s, min_time=0.000872s; KV_RETRIEVE_KEYS: req_count=7, req/min=1.32, avg_time=0.000117s, max_time=0.000138s, min_time=0.000107s (TaskRunnerV1 pid=2351) Training Progress: 98%|█████████▊| 137/140 [1:47:30<02:12, 44.01s/it] (TaskRunnerV1 pid=2351) step:135 - global_seqlen/min:139077.0 - global_seqlen/max:139077.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:139077.0 - global_seqlen/balanced_max:139077.0 - global_seqlen/mean:139077.0 - actor/entropy:0.13048124313354492 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.9952325820922852 - training/rollout_probs_diff_mean:0.004512499552220106 - training/rollout_probs_diff_std:0.016616199165582657 - training/rollout_actor_probs_pearson_corr:0.9960429668426514 - rollout_corr/training_ppl:1.1542189121246338 - rollout_corr/training_log_ppl:0.1412365883588791 - rollout_corr/kl:0.0015518658328801394 - rollout_corr/k3_kl:0.0013225176371634007 - rollout_corr/rollout_ppl:1.1522635221481323 - rollout_corr/rollout_log_ppl:0.139600932598114 - rollout_corr/log_ppl_diff:0.0016356643754988909 - rollout_corr/log_ppl_abs_diff:0.004025832284241915 - rollout_corr/log_ppl_diff_max:0.11583545058965683 - rollout_corr/log_ppl_diff_min:-0.017102181911468506 - rollout_corr/ppl_ratio:1.0016658306121826 - rollout_corr/chi2_token:0.0014728307723999023 - rollout_corr/chi2_seq:0.11410081386566162 - actor/pg_clipfrac:0.0036983045865781605 - actor/ppo_kl:-0.0003643470088074233 - actor/pg_clipfrac_lower:5.35046153042155e-05 - actor/pg_loss:0.001028774888254702 - actor/loss:0.0010287740733474493 - actor/grad_norm:2.064453125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.37581253051758 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.542) - val-core/char_count/acc/mean@1:np.float64(0.542) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:135 - training/epoch:1 - critic/score/mean:0.4794921875 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.4794921875 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.007033929694443941 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.007033929694443941 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:103.7236328125 - response_length/max:128.0 - response_length/min:39.0 - response_length/clip_ratio:0.3720703125 - response_length_non_aborted/mean:103.7236328125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:39.0 - response_length_non_aborted/clip_ratio:0.3720703125 - response/aborted_ratio:0.0 - prompt_length/mean:32.09375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:8.191657609000686 - timing_s/old_log_prob:7.830796224996448 - timing_s/adv:0.384250126000552 - timing_s/update_actor:24.94895708400145 - timing_s/update_weights:1.9175988749993849 - timing_s/step:43.44802153600176 - timing_s/testing:8.77655178400164 - timing_per_token_ms/adv:0.0027628588911218386 - timing_per_token_ms/update_actor:0.17938952583102488 - timing_per_token_ms/gen:0.07712481154849864 - perf/total_num_tokens:139077 - perf/time_per_step:43.44802153600176 - perf/throughput:3200.9973085830493 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:136 - global_seqlen/min:138635.0 - global_seqlen/max:138635.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:138635.0 - global_seqlen/balanced_max:138635.0 - global_seqlen/mean:138635.0 - actor/entropy:0.12744225561618805 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.804949939250946 - training/rollout_probs_diff_mean:0.0044587356969714165 - training/rollout_probs_diff_std:0.015383175574243069 - training/rollout_actor_probs_pearson_corr:0.9962860941886902 - rollout_corr/training_ppl:1.145958423614502 - rollout_corr/training_log_ppl:0.1343814581632614 - rollout_corr/kl:0.0010958684142678976 - rollout_corr/k3_kl:0.0009391459752805531 - rollout_corr/rollout_ppl:1.1445643901824951 - rollout_corr/rollout_log_ppl:0.13321764767169952 - rollout_corr/log_ppl_diff:0.0011638157302513719 - rollout_corr/log_ppl_abs_diff:0.003603004850447178 - rollout_corr/log_ppl_diff_max:0.05237060785293579 - rollout_corr/log_ppl_diff_min:-0.02324134111404419 - rollout_corr/ppl_ratio:1.0011767148971558 - rollout_corr/chi2_token:0.0015522241592407227 - rollout_corr/chi2_seq:0.17103397846221924 - actor/pg_clipfrac:0.003235493134222149 - actor/ppo_kl:1.0252664348323982e-05 - actor/pg_clipfrac_lower:2.5673949494375847e-05 - actor/pg_loss:0.000523653463460505 - actor/loss:0.0005236503202468157 - actor/grad_norm:1.9404296875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.43335723876953 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:136 - training/epoch:1 - critic/score/mean:0.541015625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.541015625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0002944330626633018 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0002944330626633018 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:103.1513671875 - response_length/max:128.0 - response_length/min:37.0 - response_length/clip_ratio:0.32421875 - response_length_non_aborted/mean:103.1513671875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:37.0 - response_length_non_aborted/clip_ratio:0.32421875 - response/aborted_ratio:0.0 - prompt_length/mean:32.234375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0703125 - timing_s/gen:6.207720632002747 - timing_s/old_log_prob:7.983543119000387 - timing_s/adv:0.18768177399761043 - timing_s/update_actor:24.883635558006063 - timing_s/update_weights:1.9159397399998852 - timing_s/step:41.35146619199804 - timing_per_token_ms/adv:0.0013537834890006884 - timing_per_token_ms/update_actor:0.17949028425726593 - timing_per_token_ms/gen:0.05877020678427625 - perf/total_num_tokens:138635 - perf/time_per_step:41.35146619199804 - perf/throughput:3352.601800291845 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:137 - global_seqlen/min:138213.0 - global_seqlen/max:138213.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:138213.0 - global_seqlen/balanced_max:138213.0 - global_seqlen/mean:138213.0 - actor/entropy:0.13105197250843048 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.991805911064148 - training/rollout_probs_diff_mean:0.004518910776823759 - training/rollout_probs_diff_std:0.015805557370185852 - training/rollout_actor_probs_pearson_corr:0.996220052242279 - rollout_corr/training_ppl:1.1498115062713623 - rollout_corr/training_log_ppl:0.1377224326133728 - rollout_corr/kl:0.000997038558125496 - rollout_corr/k3_kl:0.0010907636024057865 - rollout_corr/rollout_ppl:1.1486859321594238 - rollout_corr/rollout_log_ppl:0.13677218556404114 - rollout_corr/log_ppl_diff:0.0009502309840172529 - rollout_corr/log_ppl_abs_diff:0.0037968484684824944 - rollout_corr/log_ppl_diff_max:0.03639818727970123 - rollout_corr/log_ppl_diff_min:-0.022024646401405334 - rollout_corr/ppl_ratio:1.000964879989624 - rollout_corr/chi2_token:0.003397226333618164 - rollout_corr/chi2_seq:0.5009356737136841 - actor/pg_clipfrac:0.003177416103426367 - actor/ppo_kl:-7.307848025088029e-05 - actor/pg_clipfrac_lower:1.5358334470268648e-05 - actor/pg_loss:0.0019613657059380785 - actor/loss:0.0019613655749708414 - actor/grad_norm:1.794921875 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.375399589538574 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:137 - training/epoch:1 - critic/score/mean:0.5380859375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5380859375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.0048910146579146385 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.0048910146579146385 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:102.6298828125 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.3544921875 - response_length_non_aborted/mean:102.6298828125 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.3544921875 - response/aborted_ratio:0.0 - prompt_length/mean:32.34375 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0625 - timing_s/gen:6.195118714997079 - timing_s/old_log_prob:7.739855528998305 - timing_s/adv:0.3824867599978461 - timing_s/update_actor:24.77927578699746 - timing_s/update_weights:1.9273488399994676 - timing_s/step:41.195793559003505 - timing_per_token_ms/adv:0.0027673718101614616 - timing_per_token_ms/update_actor:0.17928324967258838 - timing_per_token_ms/gen:0.05894891871958246 - perf/total_num_tokens:138213 - perf/time_per_step:41.195793559003505 - perf/throughput:3355.02700784345 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0)(TaskRunnerV1 pid=2351) Training Progress: 99%|█████████▊| 138/140 [1:48:14<01:27, 43.86s/it] (SimpleStorageUnit pid=2599) 2026-11-01 19:04:29,044 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_566c4163: [Performance] Total success requests: 1078, Total req/min: 208.75, Total avg process time: 0.0006s; (SimpleStorageUnit pid=2599) Time range: last 5.16 minutes; (SimpleStorageUnit pid=2599) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.10, avg_time=0.001515s, max_time=0.004224s, min_time=0.000387s; PUT_DATA: req_count=1025, req/min=198.49, avg_time=0.000591s, max_time=0.002085s, min_time=0.000315s; GET_DATA: req_count=37, req/min=7.16, avg_time=0.000624s, max_time=0.001252s, min_time=0.000427s (TaskRunnerV1 pid=2351) Training Progress: 99%|█████████▉| 139/140 [1:48:57<00:43, 43.74s/it] (TaskRunnerV1 pid=2351) step:138 - global_seqlen/min:140300.0 - global_seqlen/max:140300.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:140300.0 - global_seqlen/balanced_max:140300.0 - global_seqlen/mean:140300.0 - actor/entropy:0.12673532962799072 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6967383623123169 - training/rollout_probs_diff_mean:0.004253033082932234 - training/rollout_probs_diff_std:0.014947302639484406 - training/rollout_actor_probs_pearson_corr:0.9964564442634583 - rollout_corr/training_ppl:1.1447256803512573 - rollout_corr/training_log_ppl:0.13324835896492004 - rollout_corr/kl:0.0011055113282054663 - rollout_corr/k3_kl:0.0008928183233365417 - rollout_corr/rollout_ppl:1.143440842628479 - rollout_corr/rollout_log_ppl:0.1321445107460022 - rollout_corr/log_ppl_diff:0.0011038504308089614 - rollout_corr/log_ppl_abs_diff:0.0035156768281012774 - rollout_corr/log_ppl_diff_max:0.03154388815164566 - rollout_corr/log_ppl_diff_min:-0.01741369068622589 - rollout_corr/ppl_ratio:1.0011159181594849 - rollout_corr/chi2_token:0.001226186752319336 - rollout_corr/chi2_seq:0.1484851837158203 - actor/pg_clipfrac:0.0027587123986450024 - actor/ppo_kl:-7.801724963959109e-05 - actor/pg_clipfrac_lower:4.4617957125107445e-05 - actor/pg_loss:0.002893248816690175 - actor/loss:0.002893249038606882 - actor/grad_norm:1.8359375 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.33707571029663 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:138 - training/epoch:1 - critic/score/mean:0.53125 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.53125 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:-0.006228421814739704 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:-0.006228421814739704 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.79296875 - response_length/max:128.0 - response_length/min:38.0 - response_length/clip_ratio:0.3671875 - response_length_non_aborted/mean:104.79296875 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:38.0 - response_length_non_aborted/clip_ratio:0.3671875 - response/aborted_ratio:0.0 - prompt_length/mean:32.21875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.03125 - timing_s/gen:8.188273644998844 - timing_s/old_log_prob:7.958419700000377 - timing_s/adv:0.3851757180018467 - timing_s/update_actor:24.7797400030031 - timing_s/update_weights:1.9136062460020185 - timing_s/step:43.39562607600237 - timing_per_token_ms/adv:0.0027453721881813735 - timing_per_token_ms/update_actor:0.17661967215255236 - timing_per_token_ms/gen:0.07630627394974135 - perf/total_num_tokens:140300 - perf/time_per_step:43.39562607600237 - perf/throughput:3233.044725620064 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) step:139 - global_seqlen/min:139473.0 - global_seqlen/max:139473.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:139473.0 - global_seqlen/balanced_max:139473.0 - global_seqlen/mean:139473.0 - actor/entropy:0.12877941131591797 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.6033223867416382 - training/rollout_probs_diff_mean:0.00435799453407526 - training/rollout_probs_diff_std:0.014973425306379795 - training/rollout_actor_probs_pearson_corr:0.9965718984603882 - rollout_corr/training_ppl:1.1506962776184082 - rollout_corr/training_log_ppl:0.13821028172969818 - rollout_corr/kl:0.0008739033946767449 - rollout_corr/k3_kl:0.0008504695142619312 - rollout_corr/rollout_ppl:1.1495988368988037 - rollout_corr/rollout_log_ppl:0.13728007674217224 - rollout_corr/log_ppl_diff:0.0009302089456468821 - rollout_corr/log_ppl_abs_diff:0.003409163560718298 - rollout_corr/log_ppl_diff_max:0.0322127491235733 - rollout_corr/log_ppl_diff_min:-0.014859497547149658 - rollout_corr/ppl_ratio:1.000941276550293 - rollout_corr/chi2_token:0.0016367435455322266 - rollout_corr/chi2_seq:0.18666136264801025 - actor/pg_clipfrac:0.0030863706075858013 - actor/ppo_kl:5.504190759590226e-05 - actor/pg_clipfrac_lower:1.3160665730538312e-05 - actor/pg_loss:0.0011852422903757542 - actor/loss:0.0011852399911731482 - actor/grad_norm:1.8720703125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.4073429107666 - actor/lr:1e-06 - perf/mfu/actor:0.0 - training/global_step:139 - training/epoch:1 - critic/score/mean:0.5224609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5224609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.004761457443237305 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.004761457443237305 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:104.0244140625 - response_length/max:128.0 - response_length/min:33.0 - response_length/clip_ratio:0.3720703125 - response_length_non_aborted/mean:104.0244140625 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:33.0 - response_length_non_aborted/clip_ratio:0.3720703125 - response/aborted_ratio:0.0 - prompt_length/mean:32.1796875 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.0546875 - timing_s/gen:8.199854576996586 - timing_s/old_log_prob:8.045933407003758 - timing_s/adv:0.18988034399808384 - timing_s/update_actor:24.766180799000722 - timing_s/update_weights:1.9174072799942223 - timing_s/step:43.28489192600682 - timing_per_token_ms/adv:0.0013614129186156735 - timing_per_token_ms/update_actor:0.17756971456124643 - timing_per_token_ms/gen:0.07697876077953254 - perf/total_num_tokens:139473 - perf/time_per_step:43.28489192600682 - perf/throughput:3222.2097317101206 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (raylet) DEBUG Command exited with code: 1 (SimpleStorageUnit pid=2583) 2026-11-01 19:04:29,044 - INFO - transfer_queue.utils.perf_utils - TQ_STORAGE_UNIT_0c74581f: [Performance] Total success requests: 1097, Total req/min: 212.43, Total avg process time: 0.0006s; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Time range: last 5.16 minutes; [repeated 7x across cluster] (SimpleStorageUnit pid=2583) Per-operation statistics: CLEAR_DATA: req_count=16, req/min=3.10, avg_time=0.001397s, max_time=0.004374s, min_time=0.000394s; PUT_DATA: req_count=1044, req/min=202.17, avg_time=0.000591s, max_time=0.002354s, min_time=0.000322s; GET_DATA: req_count=37, req/min=7.16, avg_time=0.000639s, max_time=0.001261s, min_time=0.000413s [repeated 7x across cluster] (TaskRunnerV1 pid=2351) Training Progress: 100%|██████████| 140/140 [1:49:50<00:00, 46.40s/it] Training Progress: 100%|██████████| 140/140 [1:49:50<00:00, 47.07s/it] (TaskRunnerV1 pid=2351) Traceback (most recent call last): (TaskRunnerV1 pid=2351) File "/workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/workers/default_worker.py", line 323, in (TaskRunnerV1 pid=2351) worker.main_loop() (TaskRunnerV1 pid=2351) File "/workspace/verl/.venv/lib/python3.12/site-packages/ray/_private/worker.py", line 1039, in main_loop (TaskRunnerV1 pid=2351) self.core_worker.run_task_loop() (TaskRunnerV1 pid=2351) File "/workspace/verl/.venv/lib/python3.12/site-packages/torch/utils/data/_utils/signal_handling.py", line 73, in handler (TaskRunnerV1 pid=2351) _error_if_any_worker_fails() (TaskRunnerV1 pid=2351) RuntimeError: DataLoader worker (pid 752417) is killed by signal: Killed. (raylet) DEBUG Command exited with code: 1 [repeated 26x across cluster] (TaskRunnerV1 pid=2351) step:140 - global_seqlen/min:141094.0 - global_seqlen/max:141094.0 - global_seqlen/minmax_diff:0.0 - global_seqlen/balanced_min:141094.0 - global_seqlen/balanced_max:141094.0 - global_seqlen/mean:141094.0 - actor/entropy:0.12379598617553711 - training/rollout_probs_diff_valid:1.0 - training/rollout_probs_diff_max:0.8045269846916199 - training/rollout_probs_diff_mean:0.004284057300537825 - training/rollout_probs_diff_std:0.0153504004701972 - training/rollout_actor_probs_pearson_corr:0.9963195323944092 - rollout_corr/training_ppl:1.1414508819580078 - rollout_corr/training_log_ppl:0.1304044872522354 - rollout_corr/kl:0.0011637300485745072 - rollout_corr/k3_kl:0.0009138323366641998 - rollout_corr/rollout_ppl:1.1400413513183594 - rollout_corr/rollout_log_ppl:0.12920378148555756 - rollout_corr/log_ppl_diff:0.0012006910983473063 - rollout_corr/log_ppl_abs_diff:0.0034095635637640953 - rollout_corr/log_ppl_diff_max:0.046249061822891235 - rollout_corr/log_ppl_diff_min:-0.014007017016410828 - rollout_corr/ppl_ratio:1.0012128353118896 - rollout_corr/chi2_token:0.0011894702911376953 - rollout_corr/chi2_seq:0.14809834957122803 - actor/pg_clipfrac:0.003080385911744088 - actor/ppo_kl:-4.7587289751997234e-05 - actor/pg_clipfrac_lower:3.527521857904503e-05 - actor/pg_loss:-0.0003799523401539773 - actor/loss:-0.00037995283491909504 - actor/grad_norm:2.033203125 - actor/perf/max_memory_allocated_gb:21.677851676940918 - actor/perf/max_memory_reserved_gb:28.482421875 - actor/perf/cpu_memory_used_gb:42.30944585800171 - actor/lr:1e-06 - perf/mfu/actor:0.0 - val-aux/char_count/reward/mean@1:np.float64(0.544) - val-core/char_count/acc/mean@1:np.float64(0.544) - val-aux/num_turns/min:np.int64(2) - val-aux/num_turns/max:np.int64(2) - val-aux/num_turns/mean:np.float64(2.0) - training/global_step:140 - training/epoch:1 - critic/score/mean:0.5009765625 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.5009765625 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:0.0033206907100975513 - critic/advantages/max:2.4748666286468506 - critic/advantages/min:-2.4748666286468506 - critic/returns/mean:0.0033206907100975513 - critic/returns/max:2.4748666286468506 - critic/returns/min:-2.4748666286468506 - response_length/mean:105.708984375 - response_length/max:128.0 - response_length/min:31.0 - response_length/clip_ratio:0.408203125 - response_length_non_aborted/mean:105.708984375 - response_length_non_aborted/max:128.0 - response_length_non_aborted/min:31.0 - response_length_non_aborted/clip_ratio:0.408203125 - response/aborted_ratio:0.0 - prompt_length/mean:32.078125 - prompt_length/max:40.0 - prompt_length/min:25.0 - prompt_length/clip_ratio:0.046875 - timing_s/gen:8.189311266003642 - timing_s/old_log_prob:7.915834123996319 - timing_s/adv:0.18250075099786045 - timing_s/update_actor:24.933081306997337 - timing_s/update_weights:1.9087471950042527 - timing_s/step:43.302364447998116 - timing_s/testing:9.158828156003437 - timing_per_token_ms/adv:0.0012934692545243627 - timing_per_token_ms/update_actor:0.17671255550907436 - timing_per_token_ms/gen:0.07565463172776493 - perf/total_num_tokens:141094 - perf/time_per_step:43.302364447998116 - perf/throughput:3258.3440142036593 - training/num_turns/mean:np.float64(2.0) - training/num_turns/max:np.int64(2) - training/num_turns/min:np.int64(2) - training/off_policy/trajectory_spans/mean:np.float64(1.0) - training/off_policy/trajectory_spans/max:np.int64(1) - training/off_policy/trajectory_spans/min:np.int64(1) - training/off_policy/trajectory_staleness/mean:np.float64(0.0) - training/off_policy/trajectory_staleness/max:np.int64(0) - training/off_policy/trajectory_staleness/min:np.int64(0) - training/off_policy/trajectory_staleness_worst/mean:np.float64(0.0) - training/off_policy/trajectory_staleness_worst/max:np.int64(0) - training/off_policy/trajectory_staleness_worst/min:np.int64(0) (TaskRunnerV1 pid=2351) ("Final validation metrics: {'val-aux/char_count/reward/mean@1': " (TaskRunnerV1 pid=2351) "np.float64(0.544), 'val-core/char_count/acc/mean@1': np.float64(0.544), " (TaskRunnerV1 pid=2351) "'val-aux/num_turns/min': np.int64(2), 'val-aux/num_turns/max': np.int64(2), " (TaskRunnerV1 pid=2351) "'val-aux/num_turns/mean': np.float64(2.0)}") (TaskRunnerV1 pid=2351) 🏃 View run qwen3-1.7b-char-count-grpo-direct-1gpu at: http://:5000/mlflow/#/experiments/6/runs/3cf5034225714c3db016c0fbd7dd5a52 (TaskRunnerV1 pid=2351) 🧪 View experiment at: http://:5000/mlflow/#/experiments/6