From dd0e02b244b12a65966d832053e49434c3789232 Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Sun, 22 May 2022 16:07:58 +0000 Subject: [PATCH] =?UTF-8?q?1.tokenization=5Futils=5Fbase.py=E4=B8=AD?= =?UTF-8?q?=E4=B8=80=E5=A4=84logger.warn=E6=94=B9=E4=B8=BAwarning=5Fonce?= =?UTF-8?q?=202.=20mix=5Fmodules/utils.py=20torch=E8=BD=AC=E6=8D=A2?= =?UTF-8?q?=E6=97=B6=E6=B7=BB=E5=8A=A0.cpu()=203.=20dataset=E4=B8=AD?= =?UTF-8?q?=E5=B0=86=E8=BE=93=E5=87=BA=E9=87=8D=E5=AE=9A=E5=90=91=E5=88=B0?= =?UTF-8?q?DEVNULL=E8=80=8C=E4=B8=8D=E6=98=AFNull?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/dataset/dataset.py | 3 ++- fastNLP/modules/mix_modules/utils.py | 4 ++-- fastNLP/transformers/torch/tokenization_utils_base.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fastNLP/core/dataset/dataset.py b/fastNLP/core/dataset/dataset.py index 025d33e5..cd3cae59 100644 --- a/fastNLP/core/dataset/dataset.py +++ b/fastNLP/core/dataset/dataset.py @@ -156,6 +156,7 @@ import _pickle as pickle from copy import deepcopy from typing import Optional, List, Callable, Union, Dict, Any, Mapping from types import LambdaType +from subprocess import DEVNULL import sys import time @@ -231,7 +232,7 @@ def _multi_proc(ds, _apply_field, func, counter, queue): """ idx = -1 import contextlib - with contextlib.redirect_stdout(None): # 避免打印触发 rich 的锁 + with contextlib.redirect_stdout(DEVNULL): # 避免打印触发 rich 的锁 logger.set_stdout(stdout='raw') results = [] try: diff --git a/fastNLP/modules/mix_modules/utils.py b/fastNLP/modules/mix_modules/utils.py index 142644f9..e709b0ac 100644 --- a/fastNLP/modules/mix_modules/utils.py +++ b/fastNLP/modules/mix_modules/utils.py @@ -86,12 +86,12 @@ def _torch2paddle(torch_tensor: 'torch.Tensor', device: str = None, no_gradient: if not no_gradient: # 保持梯度并保持反向传播 # paddle的stop_gradient和torch的requires_grad表现是相反的 - paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=False) + paddle_tensor = paddle.to_tensor(torch_tensor.detach().cpu().numpy(), stop_gradient=False) hook = paddle_tensor.register_hook( lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy())) ) else: - paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=True) + paddle_tensor = paddle.to_tensor(torch_tensor.detach().cpu().numpy(), stop_gradient=True) paddle_tensor = paddle_to(paddle_tensor, device) diff --git a/fastNLP/transformers/torch/tokenization_utils_base.py b/fastNLP/transformers/torch/tokenization_utils_base.py index 8ed5a2e2..3a033c96 100644 --- a/fastNLP/transformers/torch/tokenization_utils_base.py +++ b/fastNLP/transformers/torch/tokenization_utils_base.py @@ -2179,7 +2179,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): if padding is True: if verbose: if max_length is not None and (truncation is False or truncation == "do_not_truncate"): - logger.warn( + logger.warning_once( "`max_length` is ignored when `padding`=`True` and there is no truncation strategy. " "To pad to max length, use `padding='max_length'`." )