---
title: "R Notebook"
output:
  html_document:
    df_print: paged
---

# Chapter 1 R CODE: ProgramTest.docx 

## 1.3 Introductory example, and its analysis using the R statistical Software

```{r}
rm(list = ls()) # clean the global environment
```

### Reading in the data
We read the data from the directory C:\\Johannes Ledolter\\2020March01Book\\Chapter1WEB
```{r}
data <- read.csv('C:\\Johannes Ledolter\\2020March01Book\\Chapter1WEB\\test.csv', header=FALSE, stringsAsFactors=F)
```

### Cleaning the data 
```{r}
dim(data) # dimension of the data set 
data[1:4,1] # view all speeches 
```

```{r}
dim(data)[1]	# the first value of dim(data), which is the number of speeches, the data frame has 4 rows 
```

```{r}
for (i in 1:dim(data)[1]) {                                 # i takes on the value from 1 to 4
  txt=data[i,1]                                             # assigning each speech to variable 'txt'
  txt=tolower(txt)	                                        # transform all text to lower cases 
  txt=gsub("[.]","", ignore.case = TRUE,txt) 	              # delete . 
  txt=gsub("[,]","", ignore.case = TRUE,txt) 	              # delete , 
  txt=gsub("[;]","", ignore.case = TRUE,txt) 	              # delete ; 
  txt=gsub("new york","new-york", ignore.case = TRUE,txt) 	# the state of new-york
  data[i,1]=txt                                             # assign cleaned variable 'txt' back into the data frame 'data'
}

data[1:4,1]  # check new data frame 'data'
```

#### omit meta variables from the text
speaker in meta2; determine length of each speech
works if there are no missing values in meta2
```{r}
# create new variables, preparation for the for loop below
len=dim(dim(data)[1])
meta1=dim(dim(data)[1])
meta2=dim(dim(data)[1])

for (i in 1:dim(data)[1]) {
  txt=data[i,1]                  # assign each speech to variable 'txt'
  temp=strsplit(txt, " ")[[1]]   # split variable 'txt' based on space
  len[i]=length(temp)-2          # assign the length of each speech excluding the speaker name into variable 'len' 
  meta1[i]=temp[1]               # assign 1st word in 'temp'(mr/the) to variable meta1
  meta2[i]=temp[2]               # assign 2nd word in 'temp'(stevens/brooks/clerk/johnson) to variable meta2
  tempr=dim(len[i])              # create new variable 'tempr'
  for (j in 1:len[i]) { 
    tempr[j]=temp[j+2]           # assign each word in 'temp' excluding the speaker name into 'tempr' 
    }
  data[i,1]=toString(tempr)      # combine words back into one string 
  data[i,1]=gsub("[,]","", ignore.case = TRUE,data[i,1])
}

data[1:4,1] # check speeches after excluding speaker names
```

```{r}
len             # variable 'len' is the measure of how many words are in each speech after excluding speaker names 
hist(len)       # histogram of variable 'len'
boxplot(len)    # boxplot of variable 'len'
quantile(len)   # quantile of variable 'len'
meta2           # values in variable 'meta2'
```

### creating corpus

use the 'tm' package to create the corpus
```{r}
#install.packages("tm")
library(tm)

corpus <- VCorpus(VectorSource(data[,1]),readerControl = list(reader = readPlain)) # this is how to create corpus 

corpus1 <- tm_map(corpus, stripWhitespace)                                         # remove whitespace
corpus2 <- tm_map(corpus1, content_transformer(tolower))	                         # transform to lower case
corpus3 <- tm_map(corpus2, removePunctuation)                                      # remove puntuation
corpus4 <- tm_map(corpus3, removeNumbers)                                          # remove numbers
corpus5 <- tm_map(corpus4, removeWords, stopwords("english"))                      # remove stopwords
```

Document-term matrix without stemming 
```{r}
corp.dtm <- DocumentTermMatrix(corpus5,control=list(stemming=FALSE)) ## no stemming is the default
corp.dtm
```

Term-document matrix without stemming
```{r}
corp.tdm <- TermDocumentMatrix(corpus5,control=list(stemming=FALSE))
corp.tdm
```

Document-term matrix with stemming 
```{r}
corps.dtm <- DocumentTermMatrix(corpus5,control=list(stemming=TRUE))	
corps.dtm
```

```{r}
findFreqTerms(corp.dtm,1)	
```

Variable 'corps' used stemming method, here *purpose* and *purposes* are combined into *purpos*
```{r}
findFreqTerms(corps.dtm,1) 
```

#### stopwords
```{r}
stopwords("english")	
```

#### adding your own stopwords
```{r}
stopwordsnew1=c(stopwords("english"),"occasionally")  # combine 'occasionally' and 'english stopwords' to be a new stopwords set
stopwordsnew1
stopwordsnew2=c("perhaps","never")                    # 'perhaps' and 'never' are the stopwords
stopwordsnew2
```


### frequencies of words 

#### frequencies of words
```{r}
#install.packages("ggplot2")
library(ggplot2)                  # load 'ggplot2' package

dim(corp.dtm)               
```

```{r}
as.matrix(corp.dtm)
```

```{r}
findFreqTerms(corp.dtm,1)	
```

```{r}
findFreqTerms(corp.dtm,2)	
```

```{r}
freq=colSums(as.matrix(corp.dtm))     # column summation
ord=order(freq)                       # order words based on their frequencies, increasing is the default order method 
freq[head(ord)]                       # first 6 value from the variable 'ord' 
``` 

```{r}
freq[tail(ord)]                       # last 6  value from the variable 'ord'
```

```{r}
freq=sort(colSums(as.matrix(corp.dtm)),decreasing=TRUE)  
head(freq,20)
```

```{r}
wf=data.frame(word=names(freq),freq=freq)     # convert into a dataframe
head(wf)
```

#### displaying frequencies
```{r}
p=ggplot(subset(wf,freq>2),aes(word,freq))             # plot graph based on words which have more that 2 freq in 'wf'
p=p+geom_bar(stat="identity")                          # stat: statistical transformation used for the data
p=p+theme(axis.text.x=element_text(angle=45,hjust=1))  # x-axis text value is 45 degree angled, horizontally right justified 
p
```


#### displaying word clouds
```{r,fig.width = 9}
#install.packages("wordcloud")
library(wordcloud)                        # load 'wordcloud' package

set.seed(142)                             # 
wordcloud(names(freq),freq,min.freq=1)

set.seed(142)
dark2 <- brewer.pal(6,"Dark2")
wordcloud(names(freq),freq,max.words=7,rot.per=0.2,colors=dark2)
```


### finding associations
```{r}
as.matrix(corp.dtm)
```

```{r}
findAssocs(corp.dtm, "explanation", 0.5)
```

```{r}
findAssocs(corp.dtm, "gentleman", 0.5)
```

#### weightBin creates indicator variables for presence of term
```{r}
Bcorp.dtm=weightBin(corp.dtm)
as.matrix(Bcorp.dtm)
```

```{r}
findAssocs(Bcorp.dtm, "explanation", 0.5)
findAssocs(Bcorp.dtm, "gentleman", 0.5)
```


### sequence plotting
helpful as it shows whether certain terms occur together
```{r}
vecg=as.matrix(corp.dtm)[,"gentleman"]
vecg
```

```{r}
vece=as.matrix(corp.dtm)[,"explanation"]
vece
```

```{r}
par(mfrow=c(1,1))
plot(vecg,type="l",lwd=7,xlab="document",ylab="frequency",ylim=c(0,max(c(vecg,vece))))
lines(vece,type="l",col=10,lwd=3)
```


### bigrams

#### bigrams
```{r}
BigramTokenizer <-  function(x)
    unlist(lapply(ngrams(words(x), 2), paste, collapse = " "), use.names = FALSE)
bi.dtm <- DocumentTermMatrix(corpus5, control = list(tokenize = BigramTokenizer))
bi.dtm
```

```{r}
as.matrix(bi.dtm)
```

```{r}
bi.tdm <- TermDocumentMatrix(corpus5, control = list(tokenize = BigramTokenizer))
bi.tdm
```

```{r}
as.matrix(bi.tdm)
```

#### displaying bigram frequencies
```{r}
findFreqTerms(bi.dtm,1)	
findFreqTerms(bi.dtm,2)	
```

```{r}
freq=colSums(as.matrix(bi.dtm))
ord=order(freq)
freq[head(ord)]
```

```{r}
freq[tail(ord)]
```

```{r}
freq=sort(colSums(as.matrix(bi.dtm)),decreasing=TRUE)
head(freq,20)
```

```{r}
wf=data.frame(word=names(freq),freq=freq)
head(wf)
``` 

```{r}
p=ggplot(subset(wf,freq>1),aes(word,freq))
p=p+geom_bar(stat="identity")
p=p+theme(axis.text.x=element_text(angle=45,hjust=1))
p
``` 




