---
title: "combineterms"
output: html_document
---

```{r}
rm(list = ls()) # clean the global environment
```


# Chapter 1 extra code for document-term matrix

code from chapter 1 text to set up the dtm 
```{r}
data <- read.csv('C:\\Johannes Ledolter\\2020March01Book\\Chapter1WEB\\test.csv', header=FALSE, stringsAsFactors=F)

for (i in 1:dim(data)[1]) {                                   # i takes on the value from 1 to 4
  txt = data[i, 1]                                            # assigning each speech to variable 'txt'
  txt = tolower(txt)	                                        # transform all text to lower cases 
  txt = gsub("[.]", "", ignore.case = TRUE, txt) 	            # delete . 
  txt = gsub("[,]", "", ignore.case = TRUE, txt) 	            # delete , for empty space
  txt = gsub("[;]", "", ignore.case = TRUE, txt) 	            # delete ; for empty space
  txt = gsub("new york", "new-york", ignore.case = TRUE, txt) # the state of new-york
  data[i, 1] = txt                                            # assign cleaned variable 'txt' back into the data frame 'data'
}

len = dim(dim(data)[1])
meta1 = dim(dim(data)[1])
meta2 = dim(dim(data)[1])

for (i in 1:dim(data)[1]) {
  txt = data[i, 1]                  # assign each speech to variable 'txt'
  temp = strsplit(txt, " ")[[1]]    # split variable 'txt' based on space
  len[i] = length(temp) - 2         # assign the length of each speech excluding the speaker name into variable 'len' 
  meta1[i] = temp[1]                # assign 1st word in 'temp'(mr/the) to variable meta1
  meta2[i] = temp[2]                # assign 2nd word in 'temp'(stevens/brooks/clerk/johnson) to variable meta2
  tempr = dim(len[i])               # create new variable 'tempr'
  for (j in 1:len[i]) { 
    tempr[j] = temp[j + 2]          # assign each word in 'temp' excluding the speaker name into 'tempr' 
    }
  data[i, 1] = toString(tempr)      # combine words back into one string 
  data[i, 1] = gsub("[,]", "", ignore.case = TRUE, data[i, 1])
}


#install.packages("tm")
library(tm)

corpus = VCorpus(VectorSource(data[, 1]), readerControl = list(reader = readPlain)) # this is how to create corpus 

corpus1 = tm_map(corpus, stripWhitespace)                                         # remove whitespace
corpus2 = tm_map(corpus1, content_transformer(tolower))	                          # transform to lower case
corpus3 = tm_map(corpus2, removePunctuation)                                      # remove puntuation
corpus4 = tm_map(corpus3, removeNumbers)                                          # remove numbers
corpus5 = tm_map(corpus4, removeWords, stopwords("english"))                      # remove stopwords

corp.dtm = DocumentTermMatrix(corpus5, control = list(stemming = FALSE)) ## no stemming is the default
```

```{r}
corp.tdm = TermDocumentMatrix(corpus5, control = list(stemming = FALSE))
```

```{r}
corps.dtm = DocumentTermMatrix(corpus5, control = list(stemming = TRUE))
```

```{r}
corps.dtm
```

## Different ways of omitting and combing words in a document-term matrix

### OMITTING A SINGLE WORD FROM A DOCUMENT-TERM MATRIX 

```{r}
corpOLD.dtm = corp.dtm
corpOLD.dtm 
dim(corpOLD.dtm)
```

```{r}
labels(corpOLD.dtm)$Terms
ind = labels(corpOLD.dtm)$Terms == "purposes"
ind
```

```{r}
corpNEW.dtm = corpOLD.dtm[ , !ind]
dim(corpNEW.dtm)

labels(corpNEW.dtm)$Terms

as.matrix(corpNEW.dtm)
```


### OMITTING SEVERAL WORDS FROM A DOCUMENT-TERM MATRIX

```{r}
corpOLD.dtm 
dim(corpOLD.dtm)
labels(corpOLD.dtm)$Terms
```

```{r}
delete = c("floor", "proposition", "purposes")   # write down terms to delete
length(delete)
```

```{r}
corpTEMP.dtm = corpOLD.dtm
for (i in 1:length(delete)) {
  ind = labels(corpTEMP.dtm)$Terms == delete[i]
  corpTEMP.dtm = corpTEMP.dtm[ , !ind]
}
corpNEW.dtm = corpTEMP.dtm
dim(corpNEW.dtm)
```

```{r}
labels(corpNEW.dtm)$Terms
as.matrix(corpNEW.dtm)
```


### COMBINING FREQUENCIES ACROSS SEVERAL AND DELETING FREQUENCIES OF INDIVIDUAL TERMS 


combining terms in dtm matrix:	corpOLD.dtm
terms to be combined in:  	comb
new dtm matrix:			corpNEW.dtm

```{r}
library(slam)

corpOLD.dtm 
dim(corpOLD.dtm)
```

```{r}
comb = c("purpose", "purposes") 	# words to combine 
groupcomb = toString(comb) 	      # label for combined terms
mm = length(comb)
```

```{r}
corpNEW.dtm = corpOLD.dtm
h = labels(corpNEW.dtm)$Terms
ind1 = c(1:length(h))
for (j in 1:mm) {
  ind = h == comb[j]
  index = ind1[ind == TRUE]
  h[index] = groupcomb	# changing the labels of the terms to be combined to the group label
}

dimnames(corpNEW.dtm)$Terms = h	
labels(corpNEW.dtm)$Terms
corpNEW.dtm = rollup(corpNEW.dtm, 2L, labels(corpNEW.dtm)$Terms, FUN = sum)
```

```{r}
dim(corpNEW.dtm)
labels(corpNEW.dtm)$Terms
as.matrix(corpNEW.dtm)
```


### FUNCTION combine_terms TO COMBINE TERMS (COLUMNS) IN A DT MATRIX AND DELETING FREQUENCIES OF INDIVIDUAL TERMS 


combining terms in dtm matrix:	corpOLD.dtm
terms to be combined in:  	comb
new dtm matrix:			corpNEW.dtm

```{r}
combine_terms = function(corpOLD.dtm, comb) {
  
  groupcomb = toString(comb) 	# label for combined terms
  mm = length(comb)
  
  corpNEW.dtm = corpOLD.dtm
  h = labels(corpNEW.dtm)$Terms
  ind1 = c(1:length(h))
  for (j in 1:mm) {
    ind = h == comb[j]
    index = ind1[ind == TRUE]
    h[index] = groupcomb[1]	 # changing the labels of the terms to be combined to the group label
    }
  dimnames(corpNEW.dtm)$Terms = h	
  labels(corpNEW.dtm)$Terms
  corpNEW.dtm = rollup(corpNEW.dtm, 2L, labels(corpNEW.dtm)$Terms, FUN = sum)
  return(corpNEW.dtm)
}
```

```{r}
# library(slam)
corpOLD.dtm 
dim(corpOLD.dtm)
labels(corpOLD.dtm)$Terms
as.matrix(corpOLD.dtm)
```

```{r}
comb = c("purpose", "purposes") 			# words to combine 
corpNEW.dtm = combine_terms(corpOLD.dtm, comb)
dim(corpNEW.dtm)
labels(corpNEW.dtm)$Terms
as.matrix(corpNEW.dtm)
```


### ANOTHER ALTERNATIVE WAY OF COMBINING TERMS IN A DT MATRIX 


combining terms in dtm matrix:	corpOLD.dtm
terms to be combined in:  	comb
new dtm matrix:			corpNEW.dtm

```{r}
corpOLD.dtm
dim(corpOLD.dtm)
labels(corpOLD.dtm)$Terms
```

```{r}
comb = c("purpose", "purposes") 	# words to combine 
groupcomb = toString(comb) 
groupcomb			# label for combined terms
mm = length(comb)
```

```{r}
h = labels(corpOLD.dtm)$Terms
ind = h == comb[1]

for (i in 2:mm) {
  ind1 = h == comb[i]
  ind = ind|ind1
}

corpR.dtm = corpOLD.dtm[ , !ind]
dim(corpR.dtm) 
labels(corpR.dtm)$Terms
as.matrix(corpR.dtm)
```

```{r}
corpN.dtm = corpOLD.dtm[ , ind]
dim(corpN.dtm) 
if(dim(corpN.dtm)[2] > 0) {
  corpN.dtm[ , 1] = row_sums(as.matrix(corpN.dtm))	
  corpN.dtm = corpN.dtm[ , 1]
  dimnames(corpN.dtm)$Terms = groupcomb
}

corpNEW.dtm = as.DocumentTermMatrix(cbind(corpR.dtm, corpN.dtm), weighting = weightTf)   
dim(corpNEW.dtm)
labels(corpNEW.dtm)$Terms
as.matrix(corpNEW.dtm) 
```

