*** empty log message ***

hb2143 [2005-04-06 20:18:33]
*** empty log message ***
Filename
config/content extractor settings.ini
src/psl/crunch3/MainWindow.java
src/psl/crunch3/PluginFilterRunner.java
src/psl/crunch3/plugins/contentextractor/ContentExtractor.java
src/psl/crunch3/plugins/contentextractor/ContentExtractorDescriptionGUI.java
diff --git a/config/content extractor settings.ini b/config/content extractor settings.ini
index 83879f3..b59ea67 100644
--- a/config/content extractor settings.ini
+++ b/config/content extractor settings.ini
@@ -1,5 +1,5 @@
 #Content Extractor Settings File
-#Fri Feb 25 19:27:37 EST 2005
+#Wed Apr 06 16:16:47 EDT 2005
 <A>\ tags\ are\ substance=true
 <FORM>\ tags\ are\ substance=false
 Ignore\ Image\ Links=true
diff --git a/src/psl/crunch3/MainWindow.java b/src/psl/crunch3/MainWindow.java
index 6324bc8..73f360a 100644
--- a/src/psl/crunch3/MainWindow.java
+++ b/src/psl/crunch3/MainWindow.java
@@ -869,7 +869,7 @@ public class MainWindow extends Thread {
 	public void run() {
 		try {
 			//int windowHeight = 509;
-			int windowHeight = 725;
+			int windowHeight = 760;
 			//int windowWidth = 447;
 			int windowWidth = 515;

@@ -1144,24 +1144,7 @@ public class MainWindow extends Thread {
 	}

 	private void processNoGUI(){
-
-		System.out.println("working");
 		ContentExtractorDescriptionGUI dg = new ContentExtractorDescriptionGUI(mainShell);
-
-		/**TableItem selectedItem = pluginTable.getItem(0);
-
-		ProxyFilter plugin = (ProxyFilter) selectedItem.getData();
-		pluginNameLabel.setText(plugin.getName());
-		pluginDescriptionText.setText(plugin.getDescription());
-
-		if (plugin instanceof EnhancedProxyFilter){
-			EnhancedProxyFilter enhancedPlugin = (EnhancedProxyFilter)plugin;
-			pluginDescriptionGUIComposite = enhancedPlugin.getDescriptionGUI(descriptionComposite);
-		}
-		enablePluginCheck.setSelection(plugin.isEnabled());
-		configurePluginButton.setEnabled(plugin.hasSettingsGUI());**/
-
-
 	}

 	private String[] split(final String text, final String delimiters) {
diff --git a/src/psl/crunch3/PluginFilterRunner.java b/src/psl/crunch3/PluginFilterRunner.java
index 7db2451..5b31623 100644
--- a/src/psl/crunch3/PluginFilterRunner.java
+++ b/src/psl/crunch3/PluginFilterRunner.java
@@ -220,7 +220,6 @@ public class PluginFilterRunner {
 					dome.printStackTrace();
 				documentCopyable = false;
 			}
-
 		if (!documentCopyable) {
 			StringWriter stringWriter = new StringWriter();
 			XHTMLSerializer xHTMLSerializer = new XHTMLSerializer();
diff --git a/src/psl/crunch3/plugins/contentextractor/ContentExtractor.java b/src/psl/crunch3/plugins/contentextractor/ContentExtractor.java
index cc5ee18..f6948c1 100644
--- a/src/psl/crunch3/plugins/contentextractor/ContentExtractor.java
+++ b/src/psl/crunch3/plugins/contentextractor/ContentExtractor.java
@@ -16,6 +16,7 @@ import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.util.Vector;
 import java.net.URL;
+import java.net.MalformedURLException;
 import java.util.LinkedList;
 import java.util.Iterator;
 import org.w3c.dom.*;
@@ -92,6 +93,8 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	ContentExtractorDescriptionGUI descriptionGUI; // the description GUI
 	private Vector visitedClusters;
 	private boolean detectRandomSurfing = false;
+	private String linkToAppend = null;
+	private String currentAddress = null;

 	/**
 	 * Creates a new instance without any input stream and the default settings file.
@@ -136,6 +139,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 		if (Crunch3.settings.isVerbose())
 			parser.setErrorHandler(new ContentExtractorErrorHandler());

+
 		try {

 			//Create the input source using the ISO-8859-1 character set
@@ -143,8 +147,12 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 			parser.parse(new InputSource(reader));
 			mTree = parser.getDocument();
 			extractContent(mTree);
-
-		} catch (Exception e) {
+
+
+
+		}
+
+		catch (Exception e) {
 			e.printStackTrace();
 		}
 	}
@@ -153,8 +161,57 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 		if (child) {
 			if (mTree == null)
 				mTree = (Document) iNode;
-			extract(iNode);
-
+			extract(iNode, mTree);
+
+
+			org.cyberneko.html.parsers.DOMParser parser = new org.cyberneko.html.parsers.DOMParser();
+			String address = null;
+			URL site =null;
+			Document newTree;
+			InputStream in;
+			try{
+				/**if((linkToAppend !=null) && (linkToAppend != address)){
+
+					System.out.println("*** " + linkToAppend);
+					address = linkToAppend;
+					linkToAppend = null;
+					try{
+						site = new URL(address);
+					}
+					catch(MalformedURLException ex){
+						//maybe it's a relative url
+						try{
+
+							if(currentAddress == null)
+								currentAddress = Crunch3.mainWindow.getURL();
+
+								site = new URL (currentAddress + address);
+
+
+						}
+						catch(Exception e){
+							e.printStackTrace();
+							//break;
+						}
+					}
+
+					in = site.openStream();
+					InputStreamReader reader = new InputStreamReader(in, "ISO-8859-1");
+					parser.parse(new InputSource(reader));
+					newTree = parser.getDocument();
+					linkToAppend = null;
+					extract(newTree,newTree);
+					if(linkToAppend != null) System.out.println("*** " + linkToAppend);
+					//prettyPrint(newTree, System.out);
+					appendDocument(newTree, mTree);
+					//prettyPrint(mTree, System.out);
+
+				}**/
+			}
+			catch(Exception e){
+				e.printStackTrace();
+			}
+
 			//Appends the links to the bottom of the page
 			if (settings.addLinksToBottom)
 				addEnqueuedLinks();
@@ -178,12 +235,12 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 * @param iNode
 	 *            the node to start checking
 	 */
-	private void extract(final Node iNode) {
+	private void extract(final Node iNode, Document doc) {
 		NodeList children = iNode.getChildNodes();
 		if (children != null) {
 			int len = children.getLength();
 			for (int i = 0; i < len; i++) {
-				filterNode(children.item(i));
+				filterNode(children.item(i), doc);
 			}
 		}
 	}
@@ -194,16 +251,16 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 * @param iNode
 	 *            the node to filter
 	 */
-	private void filterNode(final Node iNode) {
+	private void filterNode(final Node iNode, Document doc) {
 		//Boolean that determines if the the children of the node should be
 		// filtered
 		mCheckChildren = true;

 		//Put the node through the sequence of filters
-		passThroughFilters(iNode);
+		passThroughFilters(iNode, doc);

 		if (mCheckChildren)
-			filterChildren(iNode);
+			filterChildren(iNode, doc);
 	} //filterNode

 	/**
@@ -212,7 +269,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 * @param iNode
 	 *            the node to filter
 	 */
-	private void passThroughFilters(final Node iNode) {
+	private void passThroughFilters(final Node iNode, Document doc) {
 		//Check to see if the node is a Text node or an element node and
 		//act accordingly
 		int type = iNode.getNodeType();
@@ -277,6 +334,11 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende

 			//<A HREF> with no Images
 			else if (settings.ignoreTextLinks && isTextLink(iNode) ) {
+				//before you remove, check if has a child with text "next"
+				//to append to href link to current document
+
+				linkToAppend = getNextLink(iNode);
+
 				parent.removeChild(iNode);
 				if (settings.addLinksToBottom)
 					enqueueLink(iNode);
@@ -291,7 +353,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende

 					//Make sure the image link is the image
 					if (image)
-						alt = createImageLinkAltNode(iNode);
+						alt = createImageLinkAltNode(iNode, doc);
 					if (alt != null) {
 						parent.getParentNode().insertBefore(alt, iNode.getParentNode());
 					} //if
@@ -313,7 +375,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 			//<IMG*>
 			else if (settings.ignoreImages && name.equals("IMG") && !isImageLink(iNode)) {
 				if (settings.displayImageAlts) {
-					Node alt = createAltNode(iNode);
+					Node alt = createAltNode(iNode, doc);
 					if (alt != null) {
 						//Node replaced = parent.insertBefore(alt, iNode);
 						parent.insertBefore(alt, iNode);
@@ -399,7 +461,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 			//<TABLE>
 			else if (settings.removeEmptyTables && name.equals("TABLE") ) {
 				//Call method that removes empty tables
-				removeEmptyTables(iNode);
+				removeEmptyTables(iNode, doc);
 				mCheckChildren = false;
 			} //else if

@@ -423,9 +485,8 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende

 		//Text node
 		else if (type == Node.TEXT_NODE) {
-
-			//String value = iNode.getNodeValue();
-
+
+
 			//================================================================
 			//Set of conditions determining what text to ignore
 			//================================================================
@@ -442,13 +503,13 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 * @param iNode
 	 *            the node to filter the children
 	 */
-	private void filterChildren(final Node iNode) {
+	private void filterChildren(final Node iNode, Document doc) {
 		if (iNode.hasChildNodes()) {
 			Node next = iNode.getFirstChild();
 			while (next != null) {
 				Node current = next;
 				next = current.getNextSibling();
-				filterNode(current);
+				filterNode(current, doc);
 			}
 		}
 	} //filterChildren
@@ -459,7 +520,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 * @param iNode
 	 *            the table node to examine
 	 */
-	private void removeEmptyTables(final Node iNode) {
+	private void removeEmptyTables(final Node iNode, Document doc) {
 		//First filter the children but check for
 		//undeleted nodes
 		if (iNode.hasChildNodes()) {
@@ -468,7 +529,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 			while (next != null) {
 				Node current = next;
 				next = current.getNextSibling();
-				filterNode(current);
+				filterNode(current,doc);
 			} //while
 		} //if

@@ -548,7 +609,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 *            the <IMG>node that is within the <A>tag
 	 * @return the new node or null if something went wrong
 	 */
-	private Node createImageLinkAltNode(final Node iNode) {
+	private Node createImageLinkAltNode(final Node iNode, Document doc) {
 		boolean imageMap = false;

 		//Make sure it is an image link and an image
@@ -593,21 +654,21 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 			return null;

 		//CONSTRUCT REPLACEMENT NODE
-		Element parent = mTree.createElement("B");
-		Element italic = mTree.createElement("I");
-		Element imageLink = mTree.createElement("A");
+		Element parent = doc.createElement("B");
+		Element italic = doc.createElement("I");
+		Element imageLink = doc.createElement("A");
 		imageLink.setAttribute("href", imageSource);
-		Element altLink = mTree.createElement("A");
+		Element altLink = doc.createElement("A");
 		altLink.setAttribute("href", linkHref);
-		Node openBracket = mTree.createTextNode("[");
-		Node closeBracket = mTree.createTextNode("]");
-		Node seperator = mTree.createTextNode(" | ");
+		Node openBracket = doc.createTextNode("[");
+		Node closeBracket = doc.createTextNode("]");
+		Node seperator = doc.createTextNode(" | ");
 		Node imageLinkText;
 		if (imageMap)
-			imageLinkText = mTree.createTextNode("Image Map");
+			imageLinkText = doc.createTextNode("Image Map");
 		else
-			imageLinkText = mTree.createTextNode("Image");
-		Node altLinkText = mTree.createTextNode(altTag);
+			imageLinkText = doc.createTextNode("Image");
+		Node altLinkText = doc.createTextNode(altTag);

 		//Link together nodes
 		parent.appendChild(openBracket);
@@ -630,7 +691,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 *            the image node
 	 * @return the node to add to the DOM tree or null if the node isn't an image or doesn't have an ALT attribute.
 	 */
-	private Node createAltNode(final Node iNode) {
+	private Node createAltNode(final Node iNode, Document mTree) {
 		if (!isImage(iNode))
 			return null;

@@ -832,6 +893,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 *            the node to start removing children from
 	 */
 	private void removeAll(final Node iNode) {
+		if(isTextLink(iNode)) linkToAppend = getNextLink(iNode);
 		if (isTextLink(iNode) && settings.addLinksToBottom) {
 			enqueueLink(iNode);
 		} else {
@@ -855,6 +917,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 *            the type of links to remove
 	 */
 	private void removeLinksAndText(final Node iNode, final int iType) {
+		if(isTextLink(iNode)) linkToAppend = getNextLink(iNode);
 		if (isLink(iNode) || iNode.getNodeType() == Node.TEXT_NODE) {
 			if (iType == ALL)
 				iNode.getParentNode().removeChild(iNode);
@@ -885,6 +948,22 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 		return AdsServerList.isAdsServer(iDomain);
 	} //isAdDomain

+
+	private String getNextLink(final Node iNode){
+		Node temp = iNode.getFirstChild();
+		if((temp != null) && (temp.getNodeType() == Node.TEXT_NODE)){
+
+			String text = ((temp.getNodeValue()).trim()).toLowerCase();
+			if (text.startsWith("next")){
+				String s = ((Element)iNode).getAttribute("href");
+				return s;
+			}
+		}
+		if(linkToAppend == null) return null;
+		else return linkToAppend;
+	}
+
+
 	/**
 	 * Determines if a given node contains a given attribute
 	 *
@@ -1225,6 +1304,45 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 		return false;
 	}

+
+	private void appendDocument(Document from, Document to){
+
+		//Node newNode = to.importNode(from.getDocumentElement(), true);
+		Node fromCurrent = (from.getFirstChild()).getFirstChild();
+		Node toCurrent;
+		while(fromCurrent != null){
+			System.out.println(fromCurrent.getNodeName());
+			if ((fromCurrent.getNodeName()).equals("BODY")){
+
+				toCurrent = (to.getDocumentElement()).getFirstChild();
+
+				System.out.println("before loop" + toCurrent.getNodeName());
+				while(toCurrent != null){
+					if ((toCurrent.getNodeName()).equals("BODY")){
+
+						//append all children of fromCurrent to toCurrent
+						fromCurrent = fromCurrent.getFirstChild();
+						while (fromCurrent !=null){
+							toCurrent.appendChild(to.importNode(fromCurrent, true));
+							fromCurrent = fromCurrent.getNextSibling();
+						}
+						toCurrent = null;
+					}
+					else{
+						System.out.println(toCurrent.getNodeName());
+						toCurrent = toCurrent.getNextSibling();
+					}
+				}
+
+				fromCurrent = null;
+			}
+			else fromCurrent = fromCurrent.getNextSibling();
+
+		}
+
+	}
+
+
 	/**
 	 * Add enqueued links to bottom of page
 	 */
@@ -1468,6 +1586,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 	 * @return the processed file
 	 */
 	public File process(final File in) throws IOException {
+
 		FileInputStream streamIn = new FileInputStream(in);
 		mIn = streamIn;
 		extractContent();
@@ -1623,11 +1742,13 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende

 	/**
 	 * computes extraction settings by caluculating the distance of the current URL
-	 * to the clusters generated by the preprocessor WordCount.
-	 * also checks whether a page is a homepage using the heuristics in HomePageTestser
+	 * to the clusters generated by the preprocessor WordCount.java
+	 * also checks whether a page is a homepage using the heuristics in HomePageTestser.java
 	 */
 	public void reportURL(String URL){

+		currentAddress = URL;
+
 		if (!Crunch3.settings.isGUISet()) return;
 		psl.crunch3.util.HomePageTester hpt = new psl.crunch3.util.HomePageTester(Crunch3.mainWindow.getURL().trim());
 		if (Crunch3.settings.isVerbose()){
@@ -1688,6 +1809,23 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende


 		}
+		else{
+			if(descriptionGUI.checkFrontPage() && hpt.isHomePage()){
+
+				if((descriptionGUI.getSettingsLabel()).equals("news")){
+					descriptionGUI.commitSettings("config" + File.separator + "level9.ini", 9);
+					descriptionGUI.setSettingsLevel(9);
+				}
+
+
+			}
+			else if(descriptionGUI.checkFrontPage() && !hpt.isHomePage()){
+				if((descriptionGUI.getSettingsLabel()).equals("news")){
+					descriptionGUI.commitSettings("config" + File.separator + "level2.ini", 2);
+					descriptionGUI.setSettingsLevel(2);
+				}
+			}
+		}
 	}

 	private void relax(){
@@ -1747,6 +1885,7 @@ public class ContentExtractor extends EnhancedProxyFilter implements SiteDepende
 					break;

 		}
+		descriptionGUI.updateSettingsLevel();
 	}

 	private boolean isRandomSurfing(){
diff --git a/src/psl/crunch3/plugins/contentextractor/ContentExtractorDescriptionGUI.java b/src/psl/crunch3/plugins/contentextractor/ContentExtractorDescriptionGUI.java
index a7fb346..db4f2d2 100644
--- a/src/psl/crunch3/plugins/contentextractor/ContentExtractorDescriptionGUI.java
+++ b/src/psl/crunch3/plugins/contentextractor/ContentExtractorDescriptionGUI.java
@@ -22,6 +22,7 @@ import org.eclipse.swt.SWT;
 import org.eclipse.swt.events.SelectionAdapter;
 import org.eclipse.swt.events.SelectionEvent;
 import org.eclipse.swt.graphics.Image;
+import org.eclipse.swt.layout.FillLayout;
 import org.eclipse.swt.layout.GridData;
 import org.eclipse.swt.layout.GridLayout;
 import org.eclipse.swt.widgets.Button;
@@ -48,6 +49,8 @@ public class ContentExtractorDescriptionGUI {
 	private Composite mainComposite = null;
 	private Sash ContentPluginSeparator1;
 	private Group presetsGroup;
+	private Group settingsGroup;
+	private Group userSettingsGroup;
 	private Button autoButton = null;
 	private Button newsButton = null;
 	private Button shoppingButton = null;
@@ -55,6 +58,10 @@ public class ContentExtractorDescriptionGUI {
 	private Button educationButton = null;
 	private Button textHeavyButton = null;
 	private Button linkHeavyButton = null;
+	private Button normalButton = null;
+	private Button pdaButton = null;
+	private Button impairedButton = null;
+	private Button infoButton = null;
 	private Sash ContentPluginSeparator2;
 	private Group automaticGroup;
 	private Label specificImageLabel = null;
@@ -69,6 +76,10 @@ public class ContentExtractorDescriptionGUI {
 	private Button customButton = null;
 	private Combo engineCombo = null;
 	private Scale relax;
+	private Label relaxLabel = null;
+	private Label toughenLabel = null;
+	private Button frontPageCheck = null;
+

 	private ContentExtractorSettings newFilter = ContentExtractorSettings.getInstance();
 	private boolean isAuto = false;
@@ -78,6 +89,8 @@ public class ContentExtractorDescriptionGUI {
 	private Hashtable clusters;
 	private int engineNumber = 5;
 	private int settingLevel = 0;
+	private boolean frontPage = true;
+	private String settingsLabel;

 	/**
 	 * @param c
@@ -111,7 +124,11 @@ public class ContentExtractorDescriptionGUI {
 		// init visuals
 		mainComposite = new Composite(parentComposite, SWT.NONE);
 		ContentPluginSeparator1 = new Sash(mainComposite, SWT.HORIZONTAL | SWT.BORDER);
-		presetsGroup = new Group(mainComposite, SWT.NONE);
+		settingsGroup = new Group(mainComposite, SWT.NULL);
+
+		presetsGroup = new Group(settingsGroup, SWT.NONE);
+		userSettingsGroup = new Group(settingsGroup, SWT.NULL);
+
 		newsButton = new Button(presetsGroup, SWT.RADIO);
 		shoppingButton = new Button(presetsGroup, SWT.RADIO);
 		governmentButton = new Button(presetsGroup, SWT.RADIO);
@@ -129,6 +146,13 @@ public class ContentExtractorDescriptionGUI {
 		 }


+		normalButton = new Button(userSettingsGroup, SWT.RADIO);
+		impairedButton = new Button(userSettingsGroup, SWT.RADIO);
+		infoButton = new Button(userSettingsGroup, SWT.RADIO);
+		pdaButton = new Button(userSettingsGroup, SWT.RADIO);
+		frontPageCheck = new Button(userSettingsGroup, SWT.CHECK);
+
+
 		ContentPluginSeparator2 = new Sash(mainComposite, SWT.HORIZONTAL | SWT.BORDER);
 		automaticGroup = new Group(mainComposite, SWT.NULL);
 		specificImageLabel = new Label(automaticGroup, SWT.BORDER);
@@ -140,18 +164,23 @@ public class ContentExtractorDescriptionGUI {
 		applicationLabel = new Label(referrerGroup, SWT.NULL);
 		applicationText = new Text(referrerGroup, SWT.BORDER);
 		customGroup = new Group(mainComposite, SWT.NULL);
+		toughenLabel = new Label(customGroup, SWT.NULL);
 		relax = new Scale(customGroup, SWT.NULL);
+		relaxLabel = new Label(customGroup, SWT.NULL);

-
+


 		// init nonvisuals
 		GridData mainCompositeGridData = new GridData();
 		GridLayout mainCompositeGridLayout = new GridLayout();
 		ButtonGroup modeSelectionGroup = new ButtonGroup();
+		ButtonGroup userSelectionGroup = new ButtonGroup();
 		GridData ContentSeparator1 = new GridData();
-		GridData presetsGroupGrid = new GridData();
+		GridData presetsGroupGrid = new GridData();
 		GridLayout presetsGridLayout = new GridLayout();
+		GridData userSettingsGroupGrid = new GridData();
+		GridLayout userSettingsGroupLayout = new GridLayout();
 		GridData autoButtonGridData = new GridData();
 		GridData newsButtonGridData = new GridData();
 		GridData shoppingButtonGridData = new GridData();
@@ -159,6 +188,10 @@ public class ContentExtractorDescriptionGUI {
 		GridData educationButtonGridData = new GridData();
 		GridData textHeavyButtonGridData = new GridData();
 		GridData linkHeavyButtonGridData = new GridData();
+		GridData normalButtonGridData = new GridData();
+		GridData pdaButtonGridData = new GridData();
+		GridData imparedButtonGridData = new GridData();
+		GridData infoButtonGridData = new  GridData();
 		GridData automaticGroupGrid = new GridData();
 		GridLayout automaticGridLayout = new GridLayout();
 		GridData customButtonGridData = new GridData();
@@ -175,6 +208,8 @@ public class ContentExtractorDescriptionGUI {
 		GridData customGroupGrid = new GridData();
 		GridLayout customGroupGridLayout = new GridLayout();
 		GridData comboGridData = new GridData();
+		GridData settingsGroupGridData = new GridData();
+		FillLayout settingsGroupGridLayout = new FillLayout(SWT.HORIZONTAL);


 		// set fields
@@ -190,6 +225,8 @@ public class ContentExtractorDescriptionGUI {
 		ContentSeparator2.grabExcessHorizontalSpace = true;
 		ContentSeparator2.horizontalAlignment = GridData.FILL;
 		ContentSeparator2.heightHint = 0;
+		userSettingsGroupGrid.grabExcessHorizontalSpace = true;
+		userSettingsGroupGrid.horizontalAlignment = GridData.FILL;
 		automaticGridLayout.numColumns = 3;
 		automaticGroupGrid.grabExcessHorizontalSpace = true;
 		automaticGroupGrid.horizontalAlignment = GridData.FILL;
@@ -197,7 +234,6 @@ public class ContentExtractorDescriptionGUI {
 		specificImageLabelGridData.widthHint = 100;
 		genericImageLabelGridData.heightHint = 105;
 		genericImageLabelGridData.widthHint = 100;
-		customGroupGridLayout.numColumns = 3;
 		referrerGroupGridLayout.numColumns = 2;
 		referrerGroupGrid.grabExcessHorizontalSpace = true;
 		referrerGroupGrid.horizontalAlignment = GridData.FILL;
@@ -205,9 +241,14 @@ public class ContentExtractorDescriptionGUI {
 		referrerTextGridData.horizontalAlignment = GridData.FILL;
 		applicationTextGridData.grabExcessHorizontalSpace = true;
 		applicationTextGridData.horizontalAlignment = GridData.FILL;
-		customGroupGridLayout.numColumns = 2;
+		customGroupGridLayout.numColumns = 3;
 		customGroupGrid.grabExcessHorizontalSpace = true;
 		customGroupGrid.horizontalAlignment = GridData.FILL;
+		settingsGroupGridData.grabExcessHorizontalSpace = true;
+		settingsGroupGridData.horizontalAlignment = GridData.FILL;
+		settingsGroup.setLayoutData(settingsGroupGridData);
+		settingsGroup.setLayout(settingsGroupGridLayout);
+

 		// set properties
 		mainComposite.setLayoutData(mainCompositeGridData);
@@ -242,6 +283,25 @@ public class ContentExtractorDescriptionGUI {
 		automaticGroup.setLayoutData(automaticGroupGrid);
 		automaticGroup.setLayout(automaticGridLayout);
 		automaticGroup.setText("Automatic Detection");
+
+		userSettingsGroup.setLayoutData(userSettingsGroupGrid);
+		userSettingsGroup.setLayout(userSettingsGroupLayout);
+		userSettingsGroup.setText("User Options");
+		userSelectionGroup.add(normalButton);
+		userSelectionGroup.add(infoButton);
+		userSelectionGroup.add(pdaButton);
+		userSelectionGroup.add(impairedButton);
+		normalButton.setLayoutData(normalButtonGridData);
+		normalButton.setText("Normal");
+		pdaButton.setLayoutData(pdaButtonGridData);
+		pdaButton.setText("PDA");
+		infoButton.setLayoutData(infoButtonGridData);
+		infoButton.setText("Information Retrieval");
+		impairedButton.setLayoutData(imparedButtonGridData);
+		impairedButton.setText("Visually Impaired");
+		frontPageCheck.setText("Detect Front Page");
+		frontPageCheck.setSelection(true);
+
 		ContentPluginSeparator2.setLayoutData(ContentSeparator2);
 		specificImageLabel.setLayoutData(specificImageLabelGridData);
 		specificImageLabel.setSize(100, 105);
@@ -271,6 +331,10 @@ public class ContentExtractorDescriptionGUI {
 		autoButton.setText("Automatic");
 		relax.setMaximum(12);
 		relax.setMinimum(1);
+		relax.setPageIncrement(1);
+		relaxLabel.setText("Relax");
+		toughenLabel.setText("Toughen");
+

 		// END VISUALS_INITIALIZATION

@@ -316,6 +380,30 @@ public class ContentExtractorDescriptionGUI {
 			}
 		});

+		normalButton.addSelectionListener(new SelectionAdapter() {
+			public void widgetSelected(SelectionEvent e){
+				normalButton_widgetSelected(e);
+			}
+		});
+
+		pdaButton.addSelectionListener(new SelectionAdapter() {
+			public void widgetSelected(SelectionEvent e){
+				pdaButton_widgetSelected(e);
+			}
+		});
+
+		infoButton.addSelectionListener(new SelectionAdapter() {
+			public void widgetSelected(SelectionEvent e){
+				infoButton_widgetSelected(e);
+			}
+		});
+
+		impairedButton.addSelectionListener(new SelectionAdapter() {
+			public void widgetSelected(SelectionEvent e){
+				impairedButton_widgetSelected(e);
+			}
+		});
+
 		engineCombo.addSelectionListener(new SelectionAdapter() {
 		    public void widgetSelected(SelectionEvent e){
 		      Character c = new Character((engineCombo.getText()).charAt(0));
@@ -328,62 +416,110 @@ public class ContentExtractorDescriptionGUI {
 		 	public void widgetSelected(SelectionEvent e)
 		 	{
 		 		autoButton.setSelection(false);
-		 		customButton.setSelection(true);
+		 		selectCustom();
 		 		isAuto = false;
+		 		frontPageCheck.setSelection(false);
+		 		frontPage = false;
 				commitSettings("config" + File.separator + "level" + relax.getSelection() + ".ini", relax.getSelection());
 		 	}
 			}
 		);

+		frontPageCheck.addSelectionListener(new SelectionAdapter(){
+
+			public void widgetSelected(SelectionEvent e){
+				if (frontPageCheck.getSelection() == true){
+					frontPage = true;
+				}
+				else{
+					frontPage = false;
+				}
+			}
+		});
+
 		// TODO
-		if (ContentExtractor.customLast)
+		if (ContentExtractor.customLast){
 			customButton.setSelection(true);
+			settingsLabel = "custom";
+		}
 		else{
 			newsButton.setSelection(true);
+			settingsLabel = "news";
+			relax.setSelection(2);
 		}
-		System.out.println("*************");
+
+		normalButton.setSelection(true);
+
 		if(!(Crunch3.settings.isGUISet())){
-			System.out.println("*******************" + Crunch3.settings.getSettings());
 			commitSettings(Crunch3.settings.getSettings(), 0);
 		}
 	}

 	private void newsButton_widgetSelected(SelectionEvent e) {
-		commitSettings(ContentExtractor.NEWS_SETTINGS_FILE_DEF, 0);
+		commitSettings(ContentExtractor.LEVEL2_SETTINGS_FILE_DEF, 0);
+		settingsLabel = "news";
+		relax.setSelection(1);
 		isAuto = false;
 	}

 	protected void shoppingButton_widgetSelected(SelectionEvent e) {
-		commitSettings(ContentExtractor.SHOPPING_SETTINGS_FILE_DEF , 0);
+		commitSettings(ContentExtractor.LEVEL7_SETTINGS_FILE_DEF , 0);
+		settingsLabel = "shopping";
+		relax.setSelection(7);
 		isAuto = false;
 	}

 	protected void governmentButton_widgetSelected(SelectionEvent e) {
-		commitSettings(ContentExtractor.GOVERNMENT_SETTINGS_FILE_DEF , 0);
+		commitSettings(ContentExtractor.LEVEL5_SETTINGS_FILE_DEF , 0);
+		settingsLabel = "government";
+		relax.setSelection(5);
 		isAuto = false;
 	}

 	protected void educationButton_widgetSelected(SelectionEvent e) {
-		commitSettings(ContentExtractor.EDUCATION_SETTINGS_FILE_DEF , 0);
+		commitSettings(ContentExtractor.LEVEL6_SETTINGS_FILE_DEF , 0);
+		settingsLabel = "education";
+		relax.setSelection(6);
 		isAuto = false;
 	}

 	protected void textHeavyButton_widgetSelected(SelectionEvent e) {
-		commitSettings(ContentExtractor.TEXT_HEAVY_SETTINGS_FILE_DEF , 0);
+		commitSettings(ContentExtractor.LEVEL2_SETTINGS_FILE_DEF , 0);
+		settingsLabel = "text heavy";
+		relax.setSelection(2);
 		isAuto = false;
 	}

 	protected void linkHeavyButton_widgetSelected(SelectionEvent e) {
-		commitSettings(ContentExtractor.LINK_HEAVY_SETTINGS_FILE_DEF, 0);
+		commitSettings(ContentExtractor.LEVEL10_SETTINGS_FILE_DEF, 0);
+		settingsLabel = "link heavy";
+		relax.setSelection(10);
 		isAuto = false;
 	}

+	protected void normalButton_widgetSelected(SelectionEvent e) {
+
+	}
+
+	protected void pdaButton_widgetSelected(SelectionEvent e) {
+
+	}
+
+	protected void infoButton_widgetSelected(SelectionEvent e) {
+
+	}
+
+	protected void impairedButton_widgetSelected(SelectionEvent e) {
+
+	}
+
 	/**
 	 * load automatic settings from file
 	 * @param e
 	 */
 	protected void auto_widgetSelected(SelectionEvent e) {
 		isAuto = true;
+		settingsLabel = "automatic";

 		//store cluster information
 		try{
@@ -450,6 +586,7 @@ public class ContentExtractorDescriptionGUI {
 	protected void customButton_widgetSelected(SelectionEvent e) {
 		commitSettings(ContentExtractor.CUSTOM_SETTINGS_FILE_DEF , 0);
 		isAuto = false;
+		settingsLabel = "custom";
 	}


@@ -543,6 +680,22 @@ public class ContentExtractorDescriptionGUI {
 		});
 	}

+	 public void updateSettingsLevel(){
+	 	final int level = this.settingLevel;
+	 	setSettingsLevel(level);
+
+    }
+
+	 public void setSettingsLevel(final int level){
+	 	Crunch3.Display_1.syncExec(new Runnable(){
+			public void run(){
+				if(relax != null && !relax.isDisposed())
+					relax.setSelection(level);
+			}
+		});
+    }
+
+

 	//selects the custom button
 	public void selectCustom(){
@@ -613,7 +766,19 @@ public class ContentExtractorDescriptionGUI {
     public int getSettingLevel(){
     	return settingLevel;
     }
+
+    public boolean checkFrontPage(){
+    	return frontPage;
+    }

+    public String getSettingsLabel(){
+    	return settingsLabel;
+    }
+
+
+
+
+
     /**
      * Changes the filter settings to new settings read from a file.
      * @param fileName the file containing the new filter settings.