greenfit-ai/claude-reviewed-sport-sustainability-papers

--- license: mit language: - en - it tags: - climate pretty_name: Claude Reviewed Sport Sustainability Papers size_categories: - n<1K --- # Claude Reviewed Sport Sustainability Papers ## Dataset description This dataset encompasses 16 papers (some of them divided in several parts) related to the sustainability of sports products and brands. This information lies at the core of our [application](https://huggingface.co/spaces/greenfit-ai/greenfit-ai) and will come into more intesive use with the next release of GreenFit AI. The papers were analysed with **Claude 3.5 Sonnet** (`claude-3-5-sonnet-20241022`) and they were translated into: - Their title (or a Claude-inferred title) - Their type - Their summary We also added the link to an S3 public bucket (manage via [Supabase](https://supabase.co)) were the PDFs of the papers are stored. To underline the time restrictions of our work, we also add to the dataset the time-stamps of when these papers where analysed. ## Dataset generation We generated the dataset with the following python code: ```python from sqlalchemy import MetaData, create_engine, text from sqlalchemy.orm import sessionmaker import warnings import anthropic from supabase import create_client import time import os from dotenv import load_dotenv import httpx import base64 import json from typing import List import os import time import shutil class ErrorOccuredWarning(Warning): """An error occured but it was handled by try...except""" class PGClient: def __init__(self, connection_string: str): """ Initialize a Client instance. Args: connection_string (str): A string representing the database connection information. Returns: None """ self.engine = create_engine(connection_string) self.meta = MetaData(schema="public") self.Session = sessionmaker(self.engine) with self.Session() as sess: with sess.begin(): sess.execute(text("create schema if not exists public;")) def execute_query(self, query): try: with self.Session() as sess: with sess.begin(): res = sess.execute(text(query)) return res except Exception as e: warnings.warn(f"An error occurred: {e}", ErrorOccuredWarning) return None def disconnect(self) -> None: """ Disconnect the client from the database. Returns: None """ self.engine.dispose() return load_dotenv() client = anthropic.Anthropic(api_key=os.getenv("anthropic_api_key")) supa_client = create_client(os.getenv("supa_url"), os.getenv("supa_key")) SYSTEM_V1 = """You are a PDF summarizing assistant. Based on the PDF you are provided with and on the prompt that the user gives you, you should summarize the PDF producing, as output, a JSON string with the following keys: - title (str): the title of the document - document_type (str): the type of the PDF. You can only choose among ['report', 'scientific_paper', 'thesis', 'article', 'presentation', 'other'] - summary (str): an abstract-like summary of the PDF in 100-200 words Please output only the JSON string.""" def get_pdf_url(pdf_path: str) -> str: tm = time.time() actualtm = str(tm).replace(".","") nm = os.path.basename(pdf_path)+f"_{actualtm}" with open(pdf_path, 'rb') as f: supa_client.storage.from_("PDF").upload(file=f,path=nm, file_options={"content-type": "application/pdf"}) pub_url = supa_client.storage.from_("PDF").get_public_url(nm) return pub_url def pdf_analysis(pdf_path: str) -> List[str]: pdf_url = get_pdf_url(pdf_path) pdf_data = base64.standard_b64encode(httpx.get(pdf_url).content).decode("utf-8") message = client.beta.messages.create( model="claude-3-5-sonnet-20241022", betas=["pdfs-2024-09-25"], max_tokens=1024, system=SYSTEM_V1, messages=[ { "role": "user", "content": [ { "type": "document", "source": { "type": "base64", "media_type": "application/pdf", "data": pdf_data } }, { "type": "text", "text": "Analyse the attached PDF based on your system instructions" } ] } ], ) response = message.content[0].text try: res = json.loads(response) summary = res["summary"] title = res["title"] document_type = res["document_type"] return title, document_type, summary, pdf_url except Exception as e: return "", "", "", "" def upload_to_pg_db(pgclient: PGClient, title: str, document_type: str, summary: str, pdf_url: str) -> None: title = title.replace("'","''") summary = summary.replace("'", "''") pgclient.execute_query(f"""INSERT INTO documents (title, document_type, summary, pdf_url) VALUES ('{title}', '{document_type}', '{summary}', '{pdf_url}');""") return pg_db = os.getenv("pgql_db") pg_user = os.getenv("pgql_user") pg_psw = os.getenv("pgql_psw") pg_conn_str = f"postgresql://{pg_user}:{pg_psw}@localhost:5432/{pg_db}" pg_client = PGClient(pg_conn_str) pg_client.execute_query("""CREATE TABLE IF NOT EXISTS documents ( id SERIAL PRIMARY KEY, title TEXT NOT NULL, document_type TEXT NOT NULL, summary TEXT NOT NULL, pdf_url TEXT NOT NULL, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP );""") print("Created table successfully!") pdfs = os.listdir("./data/") files = [f"./data/{pdf}" for pdf in pdfs] skipped = 0 ok = 0 s = time.time() for path in files: print(f"Analysing {path}") title, document_type, summary, pdf_url = pdf_analysis(path) if title: ok+=1 print(f"{path} correctly analysed") upload_to_pg_db(pgclient=pg_client, title=title, document_type=document_type, summary=summary, pdf_url=pdf_url) shutil.move(path, "./analysed_data/") else: skipped+=1 print("Skipping", path) e = time.time() print(f"Done in {e-s}s\nOK: {ok}\nSkipped: {skipped}") ``` ## Dataset usage The dataset is hereby provided under a **MIT License**. The dataset should be used for **scientific purposes** only.--- 许可证：MIT许可证 语言： - 英语 - 意大利语 标签： - 气候 展示名称：Claude审阅的体育可持续性论文 规模类别： - 样本数少于1000 --- # Claude审阅的体育可持续性论文 ## 数据集概况 本数据集包含16篇与体育用品及品牌可持续性相关的论文（部分论文拆分为多个子部分）。 该数据集是我们的[GreenFit AI应用](https://huggingface.co/spaces/greenfit-ai/greenfit-ai)的核心数据支撑，将在GreenFit AI的下一版本中得到更广泛的应用。 本批论文已通过**Claude 3.5 Sonnet**（模型版本标识符：`claude-3-5-sonnet-20241022`）完成分析，并为每篇论文生成以下内容： - 论文标题（或由Claude推断生成的标题） - 论文类型 - 论文摘要 我们还提供了由Supabase管理的S3公共存储桶链接，所有论文的PDF文件均存储于此。为明确标注本数据集的构建时间范围，我们同时为每条数据添加了论文的分析时间戳。 ## 数据集构建流程 我们通过以下Python代码生成本数据集： python from sqlalchemy import MetaData, create_engine, text from sqlalchemy.orm import sessionmaker import warnings import anthropic from supabase import create_client import time import os from dotenv import load_dotenv import httpx import base64 import json from typing import List import os import time import shutil class ErrorOccuredWarning(Warning): """An error occured but it was handled by try...except""" class PGClient: def __init__(self, connection_string: str): """ Initialize a Client instance. Args: connection_string (str): A string representing the database connection information. Returns: None """ self.engine = create_engine(connection_string) self.meta = MetaData(schema="public") self.Session = sessionmaker(self.engine) with self.Session() as sess: with sess.begin(): sess.execute(text("create schema if not exists public;")) def execute_query(self, query): try: with self.Session() as sess: with sess.begin(): res = sess.execute(text(query)) return res except Exception as e: warnings.warn(f"An error occurred: {e}", ErrorOccuredWarning) return None def disconnect(self) -> None: """ Disconnect the client from the database. Returns: None """ self.engine.dispose() return load_dotenv() client = anthropic.Anthropic(api_key=os.getenv("anthropic_api_key")) supa_client = create_client(os.getenv("supa_url"), os.getenv("supa_key")) SYSTEM_V1 = """You are a PDF summarizing assistant. Based on the PDF you are provided with and on the prompt that the user gives you, you should summarize the PDF producing, as output, a JSON string with the following keys: - title (str): the title of the document - document_type (str): the type of the PDF. You can only choose among ['report', 'scientific_paper', 'thesis', 'article', 'presentation', 'other'] - summary (str): an abstract-like summary of the PDF in 100-200 words Please output only the JSON string.""" def get_pdf_url(pdf_path: str) -> str: tm = time.time() actualtm = str(tm).replace(".","") nm = os.path.basename(pdf_path)+f"_{actualtm}" with open(pdf_path, 'rb') as f: supa_client.storage.from_("PDF").upload(file=f,path=nm, file_options={"content-type": "application/pdf"}) pub_url = supa_client.storage.from_("PDF").get_public_url(nm) return pub_url def pdf_analysis(pdf_path: str) -> List[str]: pdf_url = get_pdf_url(pdf_path) pdf_data = base64.standard_b64encode(httpx.get(pdf_url).content).decode("utf-8") message = client.beta.messages.create( model="claude-3-5-sonnet-20241022", betas=["pdfs-2024-09-25"], max_tokens=1024, system=SYSTEM_V1, messages=[ { "role": "user", "content": [ { "type": "document", "source": { "type": "base64", "media_type": "application/pdf", "data": pdf_data } }, { "type": "text", "text": "Analyse the attached PDF based on your system instructions" } ] } ], ) response = message.content[0].text try: res = json.loads(response) summary = res["summary"] title = res["title"] document_type = res["document_type"] return title, document_type, summary, pdf_url except Exception as e: return "", "", "", "" def upload_to_pg_db(pgclient: PGClient, title: str, document_type: str, summary: str, pdf_url: str) -> None: title = title.replace("'","''") summary = summary.replace("'", "''") pgclient.execute_query(f"INSERT INTO documents (title, document_type, summary, pdf_url) VALUES ('{title}', '{document_type}', '{summary}', '{pdf_url}');") return pg_db = os.getenv("pgql_db") pg_user = os.getenv("pgql_user") pg_psw = os.getenv("pgql_psw") pg_conn_str = f"postgresql://{pg_user}:{pg_psw}@localhost:5432/{pg_db}" pg_client = PGClient(pg_conn_str) pg_client.execute_query("CREATE TABLE IF NOT EXISTS documents ( id SERIAL PRIMARY KEY, title TEXT NOT NULL, document_type TEXT NOT NULL, summary TEXT NOT NULL, pdf_url TEXT NOT NULL, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP );") print("Created table successfully!") pdfs = os.listdir("./data/") files = ["./data/{pdf}" for pdf in pdfs] skipped = 0 ok = 0 s = time.time() for path in files: print(f"Analysing {path}") title, document_type, summary, pdf_url = pdf_analysis(path) if title: ok+=1 print(f"{path} correctly analysed") upload_to_pg_db(pgclient=pg_client, title=title, document_type=document_type, summary=summary, pdf_url=pdf_url) shutil.move(path, "./analysed_data/") else: skipped+=1 print("Skipping", path) e = time.time() print(f"Done in {e-s}s OK: {ok} Skipped: {skipped}") ## 数据集使用规范 本数据集采用**MIT许可证**进行授权，仅可用于**科学研究用途**。