A Dataset For Temporal Analysis Of Files Related To The Jfk Case

This dataset contains the content of the subset of all files with a correct publication date from the 2017 release of files related to the JFK case (retrieved from https://www.archives.gov/research/jfk/2017-release). This content was extracted from the source PDF files using the R OCR libraries tesseract and pdftools.

The code to derive the dataset is given as follows:

### BEGIN R DATA PROCESSING SCRIPT

library(tesseract)<br>
library(pdftools)

pdfs &lt;- list.files("[path to your output directory containing all PDF files]")

meta &lt;- read.csv2("[path to your input directory]/jfkrelease-2017-dce65d0ec70a54d5744de17d280f3ad2.csv",header = T,sep = ',') #the meta file containing all metadata for the PDF files (e.g. publication date)

meta$Doc.Date &lt;- as.character(meta$Doc.Date)

meta.clean &lt;- meta[-which(meta$Doc.Date=="" | grepl("/0000",meta$Doc.Date)),]<br>
for(i in 1:nrow(meta.clean)){<br>
meta.clean$Doc.Date[i] &lt;- gsub("00","01",meta.clean$Doc.Date[i])<br>
<br>
if(nchar(meta.clean$Doc.Date[i])&lt;10){<br>
meta.clean$Doc.Date[i]&lt;-format(strptime(meta.clean$Doc.Date[i],format = "%d/%m/%y"),"%m/%d/%Y")<br>
}<br>
<br>
}

meta.clean$Doc.Date &lt;- strptime(meta.clean$Doc.Date,format = "%m/%d/%Y")

meta.clean &lt;- meta.clean[order(meta.clean$Doc.Date),]

docs &lt;- data.frame(content=character(0),dpub=character(0),stringsAsFactors = F)<br>
for(i in 1:nrow(meta.clean)){<br>
#for(i in 1:3){<br>
pdf_prop &lt;- pdftools::pdf_info(paste0("[path to your output directory]/",tolower(meta.clean$File.Name[i])))<br>
tmp_files &lt;- c()<br>
for(k in 1:pdf_prop$pages){<br>
tmp_files &lt;- c(tmp_files,paste0("/home/STAFF/luczakma/RProjects/JFK/data/tmp/",k))<br>
}<br>
<br>
img_file &lt;- pdftools::pdf_convert(paste0("[path to your output directory]/",tolower(meta.clean$File.Name[i])), format = 'tiff', pages = NULL, dpi = 700,filenames = tmp_files)<br>
<br>
txt &lt;- ""<br>
<br>
for(j in 1:length(img_file)){<br>
extract &lt;- ocr(img_file[j], engine = tesseract("eng"))<br>
#unlink(img_file)<br>
txt &lt;- paste(txt,extract,collapse = " ")<br>
}<br>
<br>
docs &lt;- rbind(docs,data.frame(content=iconv(tolower(gsub("\\s+"," ",gsub("[[:punct:]]|[\n]"," ",txt))),to="UTF-8"),dpub=format(meta.clean$Doc.Date[i],"%Y/%m/%d"),stringsAsFactors = F),stringsAsFactors = F)<br>
}

<br>
write.table(docs,"[path to your output directory]/documents.csv", row.names = F)

### END R DATA PROCESSING SCRIPT