{ "componentChunkName": "component---src-templates-post-tsx", "path": "/datasets/", "result": {"data":{"logo":null,"markdownRemark":{"html":"

Huggingface Datasets Methods

\n

자주 사용하는 허깅페이스 datasets의 메서드를 정리합니다.

\n

load_datasets

\n

허깅페이스 서버에 올라가 있는 데이터셋을 다운 받을 때 사용하는 메서드

\n
>>> import datasets\n>>> dataset = datasets.load_dataset('klue', 'sts')\n>>> dataset\nDatasetDict({\n    train: Dataset({\n        features: ['guid', 'source', 'sentence1', 'sentence2', 'labels'],\n        num_rows: 11668\n    })\n    validation: Dataset({\n        features: ['guid', 'source', 'sentence1', 'sentence2', 'labels'],\n        num_rows: 519\n    })\n})
\n

save_to_disk

\n

DatasetDict Object를 디스크에 저장하는 메서드

\n
>>> import datasets\n>>> dataset = datasets.load_dataset('klue', 'sts')\n>>> dataset.save_to_disk('test')
\n

load_from_disk

\n

위의 save_to_disk로 저장한 파일을 이용해서 DatasetDict로 읽어오는 메서드

\n
>>> import datasets\n>>> dataset = datasets.load_dataset('klue', 'sts')\n>>> dataset.save_to_disk(\"path/of/my/dataset/directory\")\n>>> reloaded_encoded_dataset = datasets.load_from_disk(\"path/of/my/dataset/directory\")
\n

DatasetDict Constructor

\n

Dictionary를 바로 DatasetDict Object로 바꿔주는 방법

\n
>>> from datasets import DatasetDict\n>>> dataset = DatasetDict({\n...    \"train\": {\n...        \"test1\": [1, 2, 3, 4],\n...        \"test2\": [1, 2, 3, 4],\n...        \"test3\": [1, 2, 3, 4],\n...    },\n...    \"validation\": {\n...        \"test1\": [1, 2, 3, 4],\n...        \"test2\": [1, 2, 3, 4],\n...        \"test3\": [1, 2, 3, 4],\n...    },\n...    \"test\": {\n...        \"test1\": [1, 2, 3, 4],\n...        \"test2\": [1, 2, 3, 4],\n...        \"test3\": [1, 2, 3, 4],\n...    }\n... })\n>>> dataset\nDatasetDict({\n    train: {'test1': [1, 2, 3, 4], 'test2': [1, 2, 3, 4], 'test3': [1, 2, 3, 4]}\n    validation: {'test1': [1, 2, 3, 4], 'test2': [1, 2, 3, 4], 'test3': [1, 2, 3, 4]}\n    test: {'test1': [1, 2, 3, 4], 'test2': [1, 2, 3, 4], 'test3': [1, 2, 3, 4]}\n})
\n

from_json

\n

Json 파일로부터 DatasetDict를 로드하는 메서드

\n
>>> from datasets import DatasetDict\n>>> dataset= DatasetDict.from_json('path_of_json')\n>>> dataset\nDatasetDict({\n    train: Dataset({\n        features: ['guid', 'source', 'sentence1', 'sentence2', 'labels'],\n        num_rows: 11668\n    })\n    validation: Dataset({\n        features: ['guid', 'source', 'sentence1', 'sentence2', 'labels'],\n        num_rows: 519\n    })\n})
","htmlAst":{"type":"root","children":[{"type":"element","tagName":"h1","properties":{},"children":[{"type":"text","value":"Huggingface Datasets Methods"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"자주 사용하는 허깅페이스 datasets의 메서드를 정리합니다."}]},{"type":"text","value":"\n"},{"type":"element","tagName":"h2","properties":{},"children":[{"type":"text","value":"load_datasets"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"허깅페이스 서버에 올라가 있는 데이터셋을 다운 받을 때 사용하는 메서드"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"div","properties":{"className":["gatsby-highlight"],"dataLanguage":"python"},"children":[{"type":"element","tagName":"pre","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"code","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"import"}]},{"type":"text","value":" datasets\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset "},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":"="}]},{"type":"text","value":" datasets"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":"load_dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'klue'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sts'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset\nDatasetDict"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n train"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" Dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n features"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'guid'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'source'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sentence1'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sentence2'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'labels'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n num_rows"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"11668"}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n validation"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" Dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n features"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'guid'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'source'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sentence1'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sentence2'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'labels'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n num_rows"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"519"}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]}]}]}]},{"type":"text","value":"\n"},{"type":"element","tagName":"h2","properties":{},"children":[{"type":"text","value":"save_to_disk"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"DatasetDict Object를 디스크에 저장하는 메서드"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"div","properties":{"className":["gatsby-highlight"],"dataLanguage":"python"},"children":[{"type":"element","tagName":"pre","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"code","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"import"}]},{"type":"text","value":" datasets\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset "},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":"="}]},{"type":"text","value":" datasets"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":"load_dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'klue'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sts'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":"save_to_disk"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]}]}]}]},{"type":"text","value":"\n"},{"type":"element","tagName":"h2","properties":{},"children":[{"type":"text","value":"load_from_disk"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"위의 "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"save_to_disk"}]},{"type":"text","value":"로 저장한 파일을 이용해서 DatasetDict로 읽어오는 메서드"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"div","properties":{"className":["gatsby-highlight"],"dataLanguage":"python"},"children":[{"type":"element","tagName":"pre","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"code","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"import"}]},{"type":"text","value":" datasets\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset "},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":"="}]},{"type":"text","value":" datasets"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":"load_dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'klue'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sts'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":"save_to_disk"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"path/of/my/dataset/directory\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" reloaded_encoded_dataset "},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":"="}]},{"type":"text","value":" datasets"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":"load_from_disk"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"path/of/my/dataset/directory\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]}]}]}]},{"type":"text","value":"\n"},{"type":"element","tagName":"h2","properties":{},"children":[{"type":"text","value":"DatasetDict Constructor"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"Dictionary를 바로 DatasetDict Object로 바꿔주는 방법"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"div","properties":{"className":["gatsby-highlight"],"dataLanguage":"python"},"children":[{"type":"element","tagName":"pre","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"code","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"from"}]},{"type":"text","value":" datasets "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"import"}]},{"type":"text","value":" DatasetDict\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset "},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":"="}]},{"type":"text","value":" DatasetDict"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"train\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test1\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test2\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test3\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"validation\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test1\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test2\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test3\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test1\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test2\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"test3\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset\nDatasetDict"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n train"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test1'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test2'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test3'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"text","value":"\n validation"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test1'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test2'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test3'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"text","value":"\n test"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test1'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test2'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'test3'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"1"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"2"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]}]}]}]},{"type":"text","value":"\n"},{"type":"element","tagName":"h2","properties":{},"children":[{"type":"text","value":"from_json"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"Json 파일로부터 DatasetDict를 로드하는 메서드"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"div","properties":{"className":["gatsby-highlight"],"dataLanguage":"python"},"children":[{"type":"element","tagName":"pre","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"code","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"from"}]},{"type":"text","value":" datasets "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"import"}]},{"type":"text","value":" DatasetDict\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":"="}]},{"type":"text","value":" DatasetDict"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"."}]},{"type":"text","value":"from_json"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'path_of_json'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">>"}]},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":">"}]},{"type":"text","value":" dataset\nDatasetDict"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n train"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" Dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n features"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'guid'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'source'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sentence1'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sentence2'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'labels'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n num_rows"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"11668"}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n validation"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" Dataset"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"text","value":"\n features"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'guid'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'source'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sentence1'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'sentence2'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"'labels'"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n num_rows"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"519"}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]}]}]}]}],"data":{"quirksMode":false}},"excerpt":"Huggingface Datasets Methods 자주 사용하는 허깅페이스 datasets의 메서드를 정리합니다. load_datasets 허깅페이스 서버에 올라가 있는 데이터셋을 다운 받을 때 사용하는 메서드 save_to_disk…","fields":{"readingTime":{"text":"2 min read"}},"frontmatter":{"title":"Sooftware NLP - Huggingface Datasets Methods","userDate":"17 February 2022","date":"2022-02-17T10:00:00.000Z","tags":["nlp"],"excerpt":null,"image":{"childImageSharp":{"gatsbyImageData":{"layout":"fullWidth","backgroundColor":"#484848","images":{"fallback":{"src":"/static/df55f115c8ab295a91070882162461e4/90018/datasets.png","srcSet":"/static/df55f115c8ab295a91070882162461e4/f503b/datasets.png 750w,\n/static/df55f115c8ab295a91070882162461e4/e4953/datasets.png 1080w,\n/static/df55f115c8ab295a91070882162461e4/c2802/datasets.png 1366w,\n/static/df55f115c8ab295a91070882162461e4/90018/datasets.png 1720w","sizes":"100vw"},"sources":[{"srcSet":"/static/df55f115c8ab295a91070882162461e4/77272/datasets.webp 750w,\n/static/df55f115c8ab295a91070882162461e4/317ed/datasets.webp 1080w,\n/static/df55f115c8ab295a91070882162461e4/b5c49/datasets.webp 1366w,\n/static/df55f115c8ab295a91070882162461e4/e0b55/datasets.webp 1720w","type":"image/webp","sizes":"100vw"}]},"width":1,"height":0.5232558139534883}}},"author":[{"id":"Soohwan Kim","bio":"Co-founder/A.I. engineer at TUNiB.","avatar":{"children":[{"gatsbyImageData":{"layout":"fullWidth","backgroundColor":"#282838","images":{"fallback":{"src":"/static/a9e6b445142b247ee4cfa66155398bb2/0d6f4/soohwan.png","srcSet":"/static/a9e6b445142b247ee4cfa66155398bb2/248f9/soohwan.png 40w,\n/static/a9e6b445142b247ee4cfa66155398bb2/fd435/soohwan.png 80w,\n/static/a9e6b445142b247ee4cfa66155398bb2/0d6f4/soohwan.png 120w","sizes":"100vw"},"sources":[{"srcSet":"/static/a9e6b445142b247ee4cfa66155398bb2/e7f45/soohwan.webp 40w,\n/static/a9e6b445142b247ee4cfa66155398bb2/589ec/soohwan.webp 80w,\n/static/a9e6b445142b247ee4cfa66155398bb2/71a38/soohwan.webp 120w","type":"image/webp","sizes":"100vw"}]},"width":1,"height":0.6833333333333333}}]}}]}},"relatedPosts":{"totalCount":50,"edges":[{"node":{"id":"6529d72e-80e7-5d61-a672-d66276a3f641","excerpt":"MoE(Mixture of Experts) 기초부터 DeepSeek 혁신까지 작년 이맘때쯤 DeepSeek-V3가 저비용으로 엄청난 성능을 보이면서 화제가 되었습니다. 그 핵심 기술인 MoE(Mixture of Experts…","frontmatter":{"title":"MoE(Mixture of Experts) 기초부터 DeepSeek 혁신까지","date":"2026-01-15T01:11:55.000Z"},"fields":{"readingTime":{"text":"8 min read"},"slug":"/developed-moe/"}}},{"node":{"id":"8b49d3ef-4ce3-568c-9d71-240ff17fc3e0","excerpt":"BERT는 사실 Diffusion 모델이였다?! 최근 굉장히 흥미로운 글을 읽게되어 공유합니다. 원문 : link BERT와 Diffusion이 같은 방식이다?! NLP 연구자들에게 BERT는 너무 익숙한 모델입니다. 201…","frontmatter":{"title":"BERT는 사실 Diffusion 모델이였다?!","date":"2025-10-21T12:00:00.000Z"},"fields":{"readingTime":{"text":"8 min read"},"slug":"/bert_diffusion/"}}},{"node":{"id":"7c7e1676-ea84-58de-970c-ddec9aa10660","excerpt":"RLHF는 수다쟁이를 만든다?! (Does RLHF Breed Verbose Chatterboxes?!) RLHF(Reinforcement Learning from Human Feedback)는 OpenAI의 ChatGPT…","frontmatter":{"title":"RLHF는 수다쟁이를 만든다?! (Does RLHF Breed Verbose Chatterboxes?!)","date":"2024-03-13T01:11:55.000Z"},"fields":{"readingTime":{"text":"7 min read"},"slug":"/rlhf-vervosity/"}}},{"node":{"id":"2b9d3e22-1796-5fab-878d-5941d2e76e9d","excerpt":"LLM Paper Abstract - 2023.12 LLM…","frontmatter":{"title":"LLM Paper Abstract - 2023.12","date":"2024-01-05T10:00:00.000Z"},"fields":{"readingTime":{"text":"5 min read"},"slug":"/llm-abs-202312/"}}},{"node":{"id":"f43fa33b-917b-5c00-8462-937d99592ad7","excerpt":"What it MoE? (Mixture of Experts) 현존 최강 LLM인 GPT-4에서 “MoE (Mixture of Experts)” 방식을 채택하여 사용하고 있다고 알려졌는데요, 최근 AI계의 뜨거운 감자 Mistral AI…","frontmatter":{"title":"What is MoE? (Mixture of Experts)","date":"2023-12-22T01:11:55.000Z"},"fields":{"readingTime":{"text":"9 min read"},"slug":"/moe/"}}}]}},"pageContext":{"slug":"/datasets/","prev":{"excerpt":"Decoding Strategy (디코딩 전략) 이번 포스팅에서는 자연어처리 모델의 디코딩 전략에 관해서 다뤄보려고 합니다. 디코딩이란 말처럼 디코딩은 디코더에서\n수행하는 작업입니다. 즉, BERT와 같은 인코더 모델에서 사용하는게 아니라 GPT…","frontmatter":{"title":"Sooftware NLP - Decoding Strategy (디코딩 전략)","tags":["nlp"],"date":"2022-01-15T10:00:00.000Z","draft":false,"excerpt":null,"image":{"childImageSharp":{"gatsbyImageData":{"layout":"fullWidth","placeholder":{"fallback":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAALCAYAAAB/Ca1DAAAACXBIWXMAAAsTAAALEwEAmpwYAAACBElEQVQoz5WSbW+bUAyF+f8/ZF+mSe02Ldu6Vk2Xrk1fErIm6RryAiGBQEKABlLgcnkmaBut1fZhlo5s69jWub5WiqKgtNKX0WMGhRRk6ylibZCtDZLlhNyfkgdzCpnvel6b8oIQD+SrcRXm24Dx+R5G8z16c59B4x2RVic3m8g0/vfAStkTsQk8VnqfNE0R/ox5q8b0qoalfsW9OWTe/kKiNymy7eNrnnr/xIuBSZKyiR9Is4yikKShSxY6ZKFLGjokwYI88iru9cByWaVTnqX+Tf7/W4ESbn3urC66N2S8GuDcW6RC0L7TuOj2uezdVuhoI3qGycL3sbyMq8GG9nCDOoppafe0R1tmK4FiBzPqPw/o2yq9RRvdGxA9JHw8OWX/qE7t9IxPjTM+/zjnWO0ycVw62po3tR57x0PeHvxivz7iSA3pmxmKFy25dVUGbpfbRQfTn5BkgnpLpa7ecHjZ4vDimuPrNiedLubKYziP+fDd4Oja4dvlgkYvotFPGFopiiwky41DmicIKZDljRUg8pzZao3jB2RCkGRZhVzKavlCQvZYyjKUjO0UKQuUOI7RBhqGbqDrOkKI3YoXto1pmmiaVuE+DHdcGAZMpwaGrjMZjzAMnTzPUaIoYj6fY9kWjuOQJMnu1z3PY7lc4rpuVVPe5zMnpaxy27bxfb8SUt7hb21gPpTGByz7AAAAAElFTkSuQmCC"},"images":{"fallback":{"src":"/static/43d0a11cfbaf2b0bf2571f5678e1ced5/68f14/decoding.png","srcSet":"/static/43d0a11cfbaf2b0bf2571f5678e1ced5/d0df2/decoding.png 750w,\n/static/43d0a11cfbaf2b0bf2571f5678e1ced5/fa96f/decoding.png 1080w,\n/static/43d0a11cfbaf2b0bf2571f5678e1ced5/ad399/decoding.png 1366w,\n/static/43d0a11cfbaf2b0bf2571f5678e1ced5/68f14/decoding.png 1400w","sizes":"100vw"},"sources":[{"srcSet":"/static/43d0a11cfbaf2b0bf2571f5678e1ced5/b9516/decoding.webp 750w,\n/static/43d0a11cfbaf2b0bf2571f5678e1ced5/c4814/decoding.webp 1080w,\n/static/43d0a11cfbaf2b0bf2571f5678e1ced5/2a401/decoding.webp 1366w,\n/static/43d0a11cfbaf2b0bf2571f5678e1ced5/db739/decoding.webp 1400w","type":"image/webp","sizes":"100vw"}]},"width":1,"height":0.5614285714285715}}},"author":[{"id":"Soohwan Kim","bio":"Co-founder/A.I. engineer at TUNiB.","avatar":{"childImageSharp":{"gatsbyImageData":{"layout":"fullWidth","placeholder":{"fallback":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAOCAYAAAAvxDzwAAAACXBIWXMAABYlAAAWJQFJUiTwAAADjElEQVQ4y23Oy48TdQDA8TExEaTttvSBfdHddru7bWc6bWd+8+i0u23n0SdlBRQXZGHDgiiLDwgqXjCIJEaC8cLBmHgwUU9EYzRqQuLJi/4NJB6NF+8evgbOHr7XT76SM5ix7s4QnRHnnm9yZzvBTIswN8JcP5bC67ZoWi7CdtFsn5bloZo+dSNAMYbIIqCmD6mJIVXdR3J6I7r9IQ3T59pZlW+uxzndCbPdC/HZXoq3TikowsfqBOjOmFZ7SrM9p2kfRbXn1O0jKNbsSbJ5BMnu9Gl3+qiiz4OPSvz0QRLHFGwOZH6+neLHT5YRbRerM0J0JrTas/8FZXNKzZgg2es+ZtvDdbs8+iHJl+9lWVP7OG2br26k+PvXJCfmFqoYIpwhTXtMwxpTN0coxhjFmCKbE2rGmKoYPQY9qi2fyzsq//4e5sNXi5TlLqbd5t6lPH/9kuDTG2usqgGi7dO0hzSsIXUzQDaGKGKMLEZUxZCKCJB02+PUpMH9vQwP7ybYnSxyfNrjwumA3dEiD94v8cfnK7xzWsbpujQtH9X0qBs+iuEjC5+a7lPVPSqai9SyelyYFbjkh5hqC3TkKOutLON2nkYpgtuMcT6IszdPslo8SLlcpmUH1A0XRQyQhUtNuFQ1l4o2QFI1h3OTVXbcg+grIfSVBRqlEGopgrYSRi0e4KiIse1mScSeopA/iLADmoZL3Rgg631q+oCq1qeq95FOzhQefpzn7eMpBmqcoBVj04pzwkky0eNMRYIdL8F3twusLe2nslrE6g5pGX1U8/FhD1nvUW1tUJK7SH9+n+DR1zF2vQSvH0lz88wiV45V2J1WuHqsxLXNAnuzNJdfWmK5/BylUgFhb6DZAxR9g7rpIos+Wttja2uE9M9vYb69k+DqfJErWzLOuoVqdKlpDo4jODms4zlFckuHObx4iHwuRi6fYqmYY201x/LqMrnFDNl8gRvvukhNdT9394rcOlNFMxrUhYFmW9imTEsp0mhWKC2nWSokyWSiFA7HSKcXiEb34fayyLVniUQk8rmn2drKIqlqiNcuVrh2ymQS1Nh8sc/GoMEbl7u8eb5D1yqTzkRJJsOkkiGymQVShyLE4yHWKnEq1RjpTIhmI8r2y3mkyloI38uha1n6bpVXrkw5+4LD/S/2uHnrIopaJhZ9hmQqwqHkAXJP8BDhyD46ZoWdMxOGcx/PNfECk/8AbxXdRjRliPoAAAAASUVORK5CYII="},"images":{"fallback":{"src":"/static/a9e6b445142b247ee4cfa66155398bb2/7cf1f/soohwan.png","srcSet":"/static/a9e6b445142b247ee4cfa66155398bb2/34f77/soohwan.png 750w,\n/static/a9e6b445142b247ee4cfa66155398bb2/a94f6/soohwan.png 1080w,\n/static/a9e6b445142b247ee4cfa66155398bb2/7cf1f/soohwan.png 1148w","sizes":"100vw"},"sources":[{"srcSet":"/static/a9e6b445142b247ee4cfa66155398bb2/38420/soohwan.webp 750w,\n/static/a9e6b445142b247ee4cfa66155398bb2/7470d/soohwan.webp 1080w,\n/static/a9e6b445142b247ee4cfa66155398bb2/b5ef6/soohwan.webp 1148w","type":"image/webp","sizes":"100vw"}]},"width":1,"height":0.6829268292682927}}}}]},"fields":{"readingTime":{"text":"9 min read"},"layout":"","slug":"/generate/"}},"next":{"excerpt":"2022 AI 온라인 경진대회 1위 2022 인공지능 경진대회 - 기계독해 태스크에서 1위를 차지했습니다. 🎉 🎉 작년에는 제가 모든 리딩 및 코딩을 담당했는데, 이번에는…","frontmatter":{"title":"2022 AI 온라인 경진대회 1위","tags":["record"],"date":"2022-06-24T10:00:00.000Z","draft":false,"excerpt":null,"image":{"childImageSharp":{"gatsbyImageData":{"layout":"fullWidth","placeholder":{"fallback":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAARCAYAAADdRIy+AAAACXBIWXMAAAsTAAALEwEAmpwYAAAC40lEQVQ4y5WU7U8TWRSH5292E18BsYugsDFxvxmTdZUXxVIQCCRrXE1QoS0vFmUX2mmnVApCh05fpjN32nl5NnemQtcPKjd5cs/c8/tw7/mdM4ppmqjaIS27jVxBcE70HVwIZUdrMrtSofjFwhYOphXRsh2CwOeiS7EsQdVoUq+3MGpmSE3uhkmzaSOEi213QmQsHC/ae2K7i+v6KPOLGv2D64yMbnFnbJNbtzeISYY3GBxaZ/RehpGxbYbufCA2vMnNX1MMDqUZGZNxOozv3c/QN5giMVdAqTUCdrbT7O195p+sQUEzULUGubxBqdTg3cpH/nw0TyL+kpW/NzgoW6FmL3tCcb8R6VWdbE7ntGqjgEtZP8UNvG4V3MgMZP18fFdQq1Zo1nXwxI9r2GpZVCoGpaMWr7cafNJshBPVx/cD/OBc7PuEdfI8SfC/+CuK53XwfQfTFhSPTfS6hecJXFeEN7ywyxubJ0xOa7xYLLK8XCQxlyeeiJiOqywslVhYOmBuocTs/D7TM2qYe7Gg8XxG5Xkiz+LyPs/iKsn0MUpqXefRwzdMTGZ4PLnLxNMs41MRU89yPHiYZmjkL0Z/e83939+GZ+NSN/Uv41N7YTw9k+WPxzu8Wz2SpkDNOqITCIIeU6LnBpi1Ez5upcjvfsAyDnse1+mJvbPyKMJqUj7UEaITjpvrBVjCxWn7oSnfLmmCPJdaT5oWQncPUITToWEKLLuNabWp1gVLSYPUbjNqm1Doh0Tz+v35VtptB8tu4TgiQsi2cbCFjeyA7i+iy0+4LF365fIqA7E0A7EUV/uSXOtf41p/kis31rg1LMcxQ39si+sDaS5fXw1zN2PpMH+1b43bdze5dOU98dk8ilHz+JRZJ5sts5s7H71s3mC/VGft/Q4TTxaZn31F8u02B4ctcqpOTq2QL1TRijUKhVNyOZ1qVUQumx3pcNDjLmdPbAuTk+MyVf0Lrmj25Hqb3j/T/wc0c92EyS98KgAAAABJRU5ErkJggg=="},"images":{"fallback":{"src":"/static/961d078004cf7506560cae0b8340246e/0772c/2022-ai-compete.png","srcSet":"/static/961d078004cf7506560cae0b8340246e/4af65/2022-ai-compete.png 750w,\n/static/961d078004cf7506560cae0b8340246e/76758/2022-ai-compete.png 1080w,\n/static/961d078004cf7506560cae0b8340246e/6911c/2022-ai-compete.png 1366w,\n/static/961d078004cf7506560cae0b8340246e/0772c/2022-ai-compete.png 1718w","sizes":"100vw"},"sources":[{"srcSet":"/static/961d078004cf7506560cae0b8340246e/74aff/2022-ai-compete.webp 750w,\n/static/961d078004cf7506560cae0b8340246e/b0f5e/2022-ai-compete.webp 1080w,\n/static/961d078004cf7506560cae0b8340246e/1b8e1/2022-ai-compete.webp 1366w,\n/static/961d078004cf7506560cae0b8340246e/f7543/2022-ai-compete.webp 1718w","type":"image/webp","sizes":"100vw"}]},"width":1,"height":0.8626309662398137}}},"author":[{"id":"Soohwan Kim","bio":"Co-founder/A.I. engineer at TUNiB.","avatar":{"childImageSharp":{"gatsbyImageData":{"layout":"fullWidth","placeholder":{"fallback":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAOCAYAAAAvxDzwAAAACXBIWXMAABYlAAAWJQFJUiTwAAADjElEQVQ4y23Oy48TdQDA8TExEaTttvSBfdHddru7bWc6bWd+8+i0u23n0SdlBRQXZGHDgiiLDwgqXjCIJEaC8cLBmHgwUU9EYzRqQuLJi/4NJB6NF+8evgbOHr7XT76SM5ix7s4QnRHnnm9yZzvBTIswN8JcP5bC67ZoWi7CdtFsn5bloZo+dSNAMYbIIqCmD6mJIVXdR3J6I7r9IQ3T59pZlW+uxzndCbPdC/HZXoq3TikowsfqBOjOmFZ7SrM9p2kfRbXn1O0jKNbsSbJ5BMnu9Gl3+qiiz4OPSvz0QRLHFGwOZH6+neLHT5YRbRerM0J0JrTas/8FZXNKzZgg2es+ZtvDdbs8+iHJl+9lWVP7OG2br26k+PvXJCfmFqoYIpwhTXtMwxpTN0coxhjFmCKbE2rGmKoYPQY9qi2fyzsq//4e5sNXi5TlLqbd5t6lPH/9kuDTG2usqgGi7dO0hzSsIXUzQDaGKGKMLEZUxZCKCJB02+PUpMH9vQwP7ybYnSxyfNrjwumA3dEiD94v8cfnK7xzWsbpujQtH9X0qBs+iuEjC5+a7lPVPSqai9SyelyYFbjkh5hqC3TkKOutLON2nkYpgtuMcT6IszdPslo8SLlcpmUH1A0XRQyQhUtNuFQ1l4o2QFI1h3OTVXbcg+grIfSVBRqlEGopgrYSRi0e4KiIse1mScSeopA/iLADmoZL3Rgg631q+oCq1qeq95FOzhQefpzn7eMpBmqcoBVj04pzwkky0eNMRYIdL8F3twusLe2nslrE6g5pGX1U8/FhD1nvUW1tUJK7SH9+n+DR1zF2vQSvH0lz88wiV45V2J1WuHqsxLXNAnuzNJdfWmK5/BylUgFhb6DZAxR9g7rpIos+Wttja2uE9M9vYb69k+DqfJErWzLOuoVqdKlpDo4jODms4zlFckuHObx4iHwuRi6fYqmYY201x/LqMrnFDNl8gRvvukhNdT9394rcOlNFMxrUhYFmW9imTEsp0mhWKC2nWSokyWSiFA7HSKcXiEb34fayyLVniUQk8rmn2drKIqlqiNcuVrh2ymQS1Nh8sc/GoMEbl7u8eb5D1yqTzkRJJsOkkiGymQVShyLE4yHWKnEq1RjpTIhmI8r2y3mkyloI38uha1n6bpVXrkw5+4LD/S/2uHnrIopaJhZ9hmQqwqHkAXJP8BDhyD46ZoWdMxOGcx/PNfECk/8AbxXdRjRliPoAAAAASUVORK5CYII="},"images":{"fallback":{"src":"/static/a9e6b445142b247ee4cfa66155398bb2/7cf1f/soohwan.png","srcSet":"/static/a9e6b445142b247ee4cfa66155398bb2/34f77/soohwan.png 750w,\n/static/a9e6b445142b247ee4cfa66155398bb2/a94f6/soohwan.png 1080w,\n/static/a9e6b445142b247ee4cfa66155398bb2/7cf1f/soohwan.png 1148w","sizes":"100vw"},"sources":[{"srcSet":"/static/a9e6b445142b247ee4cfa66155398bb2/38420/soohwan.webp 750w,\n/static/a9e6b445142b247ee4cfa66155398bb2/7470d/soohwan.webp 1080w,\n/static/a9e6b445142b247ee4cfa66155398bb2/b5ef6/soohwan.webp 1148w","type":"image/webp","sizes":"100vw"}]},"width":1,"height":0.6829268292682927}}}}]},"fields":{"readingTime":{"text":"1 min read"},"layout":"","slug":"/2022_ai_online_competition/"}},"primaryTag":"nlp"}}, "staticQueryHashes": ["3170763342","3229353822"]}