PJMixers-Dev/Dampfinchen_Creative_Writing_Multiturn-8192-shrunk-l3

---
language:
- en
---
```py
import json
from tqdm import tqdm
from transformers import AutoTokenizer
import re
import pandas as pd

def load_json_or_jsonl(file_path):
try:
with open(file_path, "r") as file:
try:
# Try loading the entire file as JSON
data = json.load(file)
return data

except json.JSONDecodeError:
# If loading as JSON fails, try loading as JSON Lines
file.seek(0) # Reset file pointer to the beginning
lines = file.readlines()
json_lines_data = []

for line in lines:
try:
item = json.loads(line.strip())
json_lines_data.append(item)
except json.JSONDecodeError as e:
print(f"Error decoding JSON in line: {e}")

return json_lines_data

except FileNotFoundError:
print(f"File not found: {file_path}")
return None

def reg_check(string):
basic_slop = [
"haze of pleasure",
"finds solace in",
"reveling in the satisfaction",
"with each breath",
"a delicate dance",
"wet flesh",
"sensitive flesh",
"\\bministration(|s)\\b",
"audible pop",
"rivulets",
"admit it",
"the ball is in your court",
"the game is on",
"the choice is yours",
"i don't bite... unless you want me to",
"half-lidded eyes",
"(he|she|they) worries (his|her|their) bottom lip",
"warring with",
"arousal pooling",
"take your pleasure",
"(he|she|they) fiddles with the hem of (his|her|their) (skirt|shirt)",
"kiss-bruised lips",
"bruising kiss",
"despite (himself|herself|themselves|themself)",
"yours to take",
"\\bwanton\\b",
"reckless abandon",
"torn between",
"knuckles turning white",
"grins wickedly",
"fiery red hair",
"long lashes",
"propriety be damned",
"the world narrows",
"pupils blown wide with pleasure",
"chestnut eyes",
"(he|she|they) grasps your chin and forces you to meet (his|her|their) gaze",
"(he|she|they) bites your ear",
"nails raking angry red lines down your back",
"(her|his) cheeks flaming",
"cheeks hollowing",
"stars burst behind (his|her) eyes",
"inner walls clenching around nothing",
"puckered hole",
"wet heat",
"(he|she) whimpers, biting (his|her) lip",
"dusky nipples",
"slick fold(|s)",
"still lodged deep inside (his|her)",
"heart, body and soul belong to you",
"the night is still young",
"\\.\\.\\.for now\\b",
"whether you like it not",
"without waiting for response",
"however, (its|it is|it's) important",
"important to remember that",
"once upon",
"nestled deep within",
"an ethereal beauty",
"breathless and eager",
"whispering words of passion",
"soft and gentle",
"shivers (\\w+\\s+)?down",
"dance of pleasure",
"(his|her) sex",
"sent (shockwaves|shock waves)",
"in a rhythm",
"wild abandon",
"exhausted and spent",
"life would never be the same again",
"like an electric shock",
"threatens to consume",
"what (seemed|felt) like an eternity",
"(lay|lie) ahead",
"\\bwet pop\\b",
"maybe, just maybe",
"perhaps, just perhaps",
"starts to blur",
"but it felt like",
"unfamiliar, yet",
"moist fold(|s)",
"the night is still young",
"our shared experiences",
"bond(|s) built on mutual trust",
"the ball is in your court",
"little did (he|she|they) know",
"a pregnant silence",
"beats like a (\\w+\\s+)?drum",
"\\bpert\\b",
"for the sake of keeping things",
"her breasts heaving with desire",
"dickick",
"\\brivulets\\b",
"arousal pooling in (his|her|their) belly",
"steeling (her|him)self",
"the din of the crowd",
"journey of mutual understanding",
"revulsion warred with (reluctant|reluctance)",
"her bare mound(|s)",
"pooled around her (ankles|feet)",
"straddles your (waist|lap)",
"words turn into a purr",
"grips like a vice",
"shivers running up",
"arched spine",
"penetrated to the hilt",
"the pressure in (her|his) loins",
"catch my drift",
"sway(|s) hypnotically",
"tantalizing promise",
"with each slow, deliberate movement",
"for what (felt|seemed) like (hours|an eternity|forever)",
", but (he|she|they|I) can't help it",
"conspiratorial whisper(|s)",
"whisper(|ing) conspiratorially"
]
sus = [
"\\bloli\\b",
"\\bcunny\\b",
"\\bchild\\b",
"\\bkid\\b",
"\\btoddler\\b",
"\\binfant\\b",
"\\bbaby\\b",
"\\bkindergarten\\b",
"\\bkindergarden\\b",
]

for pattern in basic_slop:
if re.search(pattern, string.lower()):
return True

for pattern in sus:
if re.search(pattern, string.lower()):
return True

return False

def shrink_sharegpt(
sharegpt_file,
output_file,
max_length
):
# Subtract 2 to allow room for BOS and EOS
max_length = max_length - 2

json_data = []

sharegpt_data = load_json_or_jsonl(sharegpt_file)
for sample_index, sample in tqdm(pd.DataFrame(sharegpt_data).iterrows(), total=len(sharegpt_data)):
sample_length = 0
new_sample_data = []
for turn in sample["conversations"]:
if turn["from"] == "system":
turn_name = "system"
elif turn["from"] == "human":
turn_name = "user"
elif turn["from"] == "gpt":
turn_name = "assistant"
else:
print("Unknown 'from'")
exit()

# Skip any samples which contain stuff we don't want
if reg_check(turn['value']):
new_sample_data = []
break

turn_length = len(
tokenizer(
f"<|start_header_id|>{turn_name}<|end_header_id|>\n\n"
f"{turn['value']}<|eot_id|>",
add_special_tokens=False
)["input_ids"]
)

if sample_length + turn_length <= max_length:
sample_length += turn_length
new_sample_data.append(turn)
else:
break

# Check if there's less than 2 turns
if len(new_sample_data) < 2:
continue

# Don't end on a user turn
while new_sample_data[-1]["from"] == "human":
del new_sample_data[-1]

# Again check if there's less than 2 turns, this time after possibly removing 'human' turns
if len(new_sample_data) < 2:
continue

json_data.append({"conversations": new_sample_data})

df = pd.DataFrame(json_data)
df.to_parquet(output_file, index=False)

if __name__ == "__main__":
source_file = "./downloaded_datasets/Creative_Writing_Multiturn.json"
output_file = "./downloaded_datasets/Creative_Writing_Multiturn-8192-shrunk-l3.parquet"
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")

shrink_sharegpt(
source_file,
output_file,
max_length=8192
)
```

语言：英语

以下为一段用于预处理ShareGPT格式数据集的Python脚本：
1. 工具函数`load_json_or_jsonl`：用于加载JSON或JSON Lines格式的数据集文件，若文件不存在则打印提示并返回空值；若文件无法以标准JSON格式加载，则尝试以JSON Lines逐行解析。
2. 正则过滤函数`reg_check`：通过预设的正则表达式规则集合，检测输入文本是否包含不良或敏感内容，包含两个规则组：
- `basic_slop`：基础不良内容匹配规则
- `sus`：可疑敏感内容匹配规则
3. 核心处理函数`shrink_sharegpt`：实现数据集的清洗与截断流程：
- 加载输入的ShareGPT格式数据集
- 遍历每一条对话样本，根据对话轮次的`from`字段将角色映射为标准格式：`system`对应系统提示、`human`对应用户、`gpt`对应助手
- 逐轮检查对话内容是否包含违规内容，若存在则直接跳过该样本
- 使用自动分词器（AutoTokenizer）计算对话的Token长度，根据指定的最大上下文长度截断过长的对话，预留序列起始标记（BOS）与序列结束标记（EOS）的位置
- 过滤掉轮次不足2条的样本，且确保对话最后一轮为助手回复，避免以用户提问结尾
- 将处理后的有效对话样本整理为标准格式，并保存为Parquet格式文件
4. 主函数部分：指定输入数据集路径为`./downloaded_datasets/Creative_Writing_Multiturn.json`，输出处理后的数据集路径为`./downloaded_datasets/Creative_Writing_Multiturn-8192-shrunk-l3.parquet`，并从`meta-llama/Meta-Llama-3-8B-Instruct`模型加载自动分词器，调用核心处理函数完成数据集预处理，最大上下文长度设置为8192。

代码依赖的库包括：进度条库`tqdm`、Hugging Face Transformers库的自动分词器（AutoTokenizer）、正则表达式库`re`以及Pandas数据处理库。