pjin/hross/mt-verifiers-main-ray-dev-20251208 vs main

.gitignore

@@ -16,7 +16,7 @@ fastspeech_output
 .bash_history.local
 
 # Byte-compiled / optimized / DLL files
-__pycache__/
+**/__pycache__/
 *.py[cod]
 *$py.class
 **.pyc

nemo_gym/config_types.py

@@ -312,6 +312,7 @@ class DatasetConfig(BaseModel):
             Literal["MIT"],
             Literal["Creative Commons Attribution 4.0 International"],
             Literal["Creative Commons Attribution-ShareAlike 4.0 International"],
+            Literal["NVIDIA Internal Use Only, Do Not Distribute"],
             Literal["TBD"],
             Literal["MIT"],
         ]

resources_servers/translation_bleu/README.md

@@ -0,0 +1,11 @@
+# Description
+
+Data links: ?
+
+# Licensing information
+Code: Apache 2.0
+Data: NVIDIA Internal Use Only, Do Not Distribute
+
+Dependencies
+- nemo_gym: Apache 2.0
+- sacrebleu: Apache 2.0

resources_servers/translation_bleu/app.py

@@ -0,0 +1,110 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Dict
+
+from fastapi import FastAPI
+from sacrebleu.metrics import BLEU
+
+from nemo_gym.base_resources_server import (
+    BaseResourcesServerConfig,
+    BaseVerifyRequest,
+    BaseVerifyResponse,
+    SimpleResourcesServer,
+)
+
+
+class TranslationBleuResourcesServerConfig(BaseResourcesServerConfig):
+    reasoning_split_word: str = "</think>"
+
+
+class TranslationBleuVerifyRequest(BaseVerifyRequest):
+    trg_txt: str
+    trg_lang: str
+
+
+class TranslationBleuVerifyResponse(BaseVerifyResponse):
+    trg_txt: str
+    trg_lang: str
+    extracted_answer: str
+
+
+class TranslationBleuResourcesServer(SimpleResourcesServer):
+    config: TranslationBleuResourcesServerConfig
+
+    TOKENIZER_MAP: Dict[str, str] = {
+        "zh": "zh",
+        "zh-cn": "zh",
+        "zh-tw": "zh",
+        "zho-CN": "zh",
+        "zho_simpl": "zh",
+        "ja": "ja-mecab",
+        "jpn": "ja-mecab",
+        "th": "flores200",
+        "ko": "ko-mecab",
+    }
+
+    def setup_webserver(self) -> FastAPI:
+        app = super().setup_webserver()
+
+        # Additional server routes go here! e.g.:
+        # app.post("/get_weather")(self.get_weather)
+
+        return app
+
+    async def verify(self, body: TranslationBleuVerifyRequest) -> TranslationBleuVerifyResponse:
+        assistant_responses = []
+        for output_item in body.response.output:
+            if output_item.type != "message":
+                continue
+
+            for content_item in output_item.content:
+                if content_item.type != "output_text":
+                    continue
+
+                assistant_responses.append(content_item.text)
+
+        combined_response = "".join(assistant_responses)
+
+        (reward, extracted_answer) = self._verify_answer(
+            ground_truth=body.trg_txt, target_lang=body.trg_lang, model_response=combined_response
+        )
+
+        return TranslationBleuVerifyResponse(**body.model_dump(), extracted_answer=extracted_answer, reward=reward)
+
+    def _verify_answer(self, ground_truth: str, target_lang: str, model_response: str) -> tuple[float, str]:
+        extracted_answer = self._extract_answer(model_response)
+
+        if target_lang in self.TOKENIZER_MAP:
+            tokenize = self.TOKENIZER_MAP[target_lang]
+        else:
+            tokenize = None
+        # Use effective_order for sentence-level BLEU
+        bleu = BLEU(trg_lang=target_lang, effective_order=True, tokenize=tokenize)
+
+        bleu_output = bleu.sentence_score(extracted_answer, [ground_truth])
+        # TODO Do we want to report any other BLEU outputs?
+        bleu_score = bleu_output.score
+        reward = bleu_score / 100.0
+
+        return reward, extracted_answer
+
+    def _extract_answer(self, model_response: str) -> str:
+        # Strip any thinking
+        no_think_response = model_response.split(self.config.reasoning_split_word)[-1]
+        no_think_response = no_think_response.strip()
+        return no_think_response
+
+
+if __name__ == "__main__":
+    TranslationBleuResourcesServer.run_webserver()

resources_servers/translation_bleu/configs/translation_bleu.yaml

@@ -0,0 +1,38 @@
+translation_bleu:
+  resources_servers:
+    translation_bleu:
+      entrypoint: app.py
+      domain: translation
+translation_bleu_simple_agent:
+  responses_api_agents:
+    simple_agent:
+      entrypoint: app.py
+      resources_server:
+        type: resources_servers
+        name: translation_bleu
+      model_server:
+        type: responses_api_models
+        name: policy_model
+      datasets:
+      - name: train
+        type: train
+        jsonl_fpath: resources_servers/translation_bleu/data/riva_mt_v3_nothinkInSys_train.jsonl
+        num_repeats: 1
+        gitlab_identifier:
+          dataset_name: riva_mt_v3_nothinkInSys_train
+          version: 0.0.3
+          artifact_fpath: riva_mt_v3_nothinkInSys_train.jsonl
+        license: NVIDIA Internal Use Only, Do Not Distribute
+      - name: validation
+        type: validation
+        jsonl_fpath: resources_servers/translation_bleu/data/riva_mt_v3_nothinkInSys_validation.jsonl
+        num_repeats: 1
+        gitlab_identifier:
+          dataset_name: riva_mt_v3_nothinkInSys_validation
+          version: 0.0.1
+          artifact_fpath: riva_mt_v3_nothinkInSys_validation.jsonl
+        license: NVIDIA Internal Use Only, Do Not Distribute
+      - name: example
+        type: example
+        jsonl_fpath: resources_servers/translation_bleu/data/example.jsonl
+        num_repeats: 1

resources_servers/translation_bleu/data/.gitignore

@@ -0,0 +1,5 @@
+*train.jsonl
+*validation.jsonl
+*train_prepare.jsonl
+*validation_prepare.jsonl
+*example_prepare.jsonl

resources_servers/translation_bleu/data/example.jsonl

@@ -0,0 +1,5 @@
+{"responses_create_params": {"input": [{"role": "user", "content": "Convert the following text into English. Text:\n```\n¿Por qué malgastar el tiempo buscando las llaves?\n```"}]}, "src_txt": "¿Por qué malgastar el tiempo buscando las llaves?", "trg_txt": "Why waste your time looking for the keys?", "src_lang": "es-us", "trg_lang": "en"}
+{"responses_create_params": {"input": [{"role": "user", "content": "Translate this into English: Ihm wurden das Konservatorium und das Theater gewidmet, in dem jedes Jahr das „Rossini Opera Festival“ stattfindet, das Opernliebhaber aus aller Welt anlockt."}]}, "src_txt": "Ihm wurden das Konservatorium und das Theater gewidmet, in dem jedes Jahr das „Rossini Opera Festival“ stattfindet, das Opernliebhaber aus aller Welt anlockt.", "trg_txt": "Every year, the conservatory and theatre, both named after him, host the Rossini Opera Festival, drawing enthusiasts from around the world.", "src_lang": "de", "trg_lang": "en"}
+{"responses_create_params": {"input": [{"role": "user", "content": "Convert the following text into French. Text:\n```\nAll questions have been correctly answered by the treasurer Thomas Kräuchi.\n```"}]}, "src_txt": "All questions have been correctly answered by the treasurer Thomas Kräuchi.", "trg_txt": "Le trésorier Thomas Kräuchi a répondu correctement à toutes les questions.", "src_lang": "en", "trg_lang": "fr"}
+{"responses_create_params": {"input": [{"role": "user", "content": "Convert the following text into Japanese. Text:\n```\nThe next picture shows the atoms emitting photons. Of course, in reality photons are a lot smaller than those in the picture.\n```"}]}, "src_txt": "The next picture shows the atoms emitting photons. Of course, in reality photons are a lot smaller than those in the picture.", "trg_txt": "次の写真は、原子が光子を放出している様子です。もちろん、実際には写真よりもはるかに微小です。", "src_lang": "en", "trg_lang": "ja"}
+{"responses_create_params": {"input": [{"role": "user", "content": "Translate the following text from Simplified Chinese to English: 比赛是在草地上进行的，洞周围的草被修剪得更短，被称为果岭。"}]}, "src_txt": "比赛是在草地上进行的，洞周围的草被修剪得更短，被称为果岭。", "trg_txt": "The game is played on grass, and the grass around the hole is mown shorter and called the green.", "src_lang": "zh-cn", "trg_lang": "en"}

resources_servers/translation_bleu/data/example_nothink.jsonl

@@ -0,0 +1,5 @@
+{"responses_create_params": {"input": [{"role": "system", "content":"/no_think"}, {"role": "user", "content": "Convert the following text into English. Text:\n```\n¿Por qué malgastar el tiempo buscando las llaves?\n```"}]}, "src_txt": "¿Por qué malgastar el tiempo buscando las llaves?", "trg_txt": "Why waste your time looking for the keys?", "src_lang": "es-us", "trg_lang": "en"}
+{"responses_create_params": {"input": [{"role": "system", "content":"/no_think"}, {"role": "user", "content": "Translate this into English: Ihm wurden das Konservatorium und das Theater gewidmet, in dem jedes Jahr das „Rossini Opera Festival“ stattfindet, das Opernliebhaber aus aller Welt anlockt."}]}, "src_txt": "Ihm wurden das Konservatorium und das Theater gewidmet, in dem jedes Jahr das „Rossini Opera Festival“ stattfindet, das Opernliebhaber aus aller Welt anlockt.", "trg_txt": "Every year, the conservatory and theatre, both named after him, host the Rossini Opera Festival, drawing enthusiasts from around the world.", "src_lang": "de", "trg_lang": "en"}
+{"responses_create_params": {"input": [{"role": "system", "content":"/no_think"}, {"role": "user", "content": "Convert the following text into French. Text:\n```\nAll questions have been correctly answered by the treasurer Thomas Kräuchi.\n```"}]}, "src_txt": "All questions have been correctly answered by the treasurer Thomas Kräuchi.", "trg_txt": "Le trésorier Thomas Kräuchi a répondu correctement à toutes les questions.", "src_lang": "en", "trg_lang": "fr"}
+{"responses_create_params": {"input": [{"role": "system", "content": "/no_think"}, {"role": "user", "content": "Convert the following text into Japanese. Text:\n```\nThe next picture shows the atoms emitting photons. Of course, in reality photons are a lot smaller than those in the picture.\n```"}]}, "src_txt": "The next picture shows the atoms emitting photons. Of course, in reality photons are a lot smaller than those in the picture.", "trg_txt": "次の写真は、原子が光子を放出している様子です。もちろん、実際には写真よりもはるかに微小です。", "src_lang": "en", "trg_lang": "ja"}
+{"responses_create_params": {"input": [{"role": "system", "content": "/no_think"}, {"role": "user", "content": "Translate the following text from Simplified Chinese to English: 比赛是在草地上进行的，洞周围的草被修剪得更短，被称为果岭。"}]}, "src_txt": "比赛是在草地上进行的，洞周围的草被修剪得更短，被称为果岭。", "trg_txt": "The game is played on grass, and the grass around the hole is mown shorter and called the green.", "src_lang": "zh-cn", "trg_lang": "en"}