package example.hector;

import static me.prettyprint.cassandra.utils.StringUtils.bytes;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import me.prettyprint.cassandra.service.CassandraClient;
import me.prettyprint.cassandra.service.CassandraClientPool;
import me.prettyprint.cassandra.service.CassandraClientPoolFactory;
import me.prettyprint.cassandra.service.Keyspace;
import me.prettyprint.cassandra.service.PoolExhaustedException;

import org.apache.cassandra.thrift.Column;
import org.apache.cassandra.thrift.ColumnParent;
import org.apache.cassandra.thrift.ColumnPath;
import org.apache.cassandra.thrift.SlicePredicate;

/**
 * Simple example showing how to iterate over an entire column family. This will work even with
 * the Random Partitioner but be aware that you will of course, get results in an unordered manner.
 * @author mark j greene
 */
public class IterateExample {

public IterateExample() {
}

private void iterateEntireColumnFamily(CassandraClient client, Keyspace ks, ColumnParent cp,
				SlicePredicate sp, String start, String end, int rowFetchSize) throws Exception {

	// get all the keys with a limit of rowFetchSize values returned
	Map<String, List<Column>> results = this.fetchPagedResults(ks, cp, sp, start, end, rowFetchSize);

	Iterator<String> i = results.keySet().iterator();
	while (i.hasNext()) {
		String key = i.next();
		List<Coulmn> c = results.get(key);

		//check for tombstones
		if (c.isEmpty()) {
			System.out.println("Column is null for key: " + key);
		} else {
			//key was found, print to console
			System.out.println(key + " -> " + new String(results.get(key).get(0).getValue()));
		}

		if(!i.hasNext()) {
			System.out.println("This page has run out, getting more results from key: " + key);

			//we've run out of results in this page, get more
			results = this.fetchPagedResults(ks, cp, sp, key, "", rowFetchSize);

			String inclusiveStart = results.keySet().iterator().next();
			System.out.println("Removing inclusive start key: " + inclusiveStart);

			//remove the inclusive start key from the results
			results.remove(inclusiveStart);
			Set<String> keySet = results.keySet();

			//if more results were found, replace the iterator
			if(keySet.size() > 0) {
				i = keySet.iterator();
			}
		}
	}
}

private Map<String, List<Column>> fetchPagedResults( Keyspace ks, ColumnParent cp,
		SlicePredicate sp, String start, String end, int rowFetchSize ) throws Exception {

	 Map<String, List<Column>> results = ks.getRangeSlice(cp, sp, start, end, rowFetchSize);		

	return results;
}

public void setupAndIterate() throws Exception {
	CassandraClientPool pool = CassandraClientPoolFactory.INSTANCE.get();
	CassandraClient client = pool.borrowClient("localhost", 9160);

	try {
		Keyspace ks = client.getKeyspace("Keyspace1");

		// just look for one column for now
		List<byte[]> colNames = new ArrayList<byte[]>();
		colNames.add("column-name".getBytes());

		// setup the slice predicate
		SlicePredicate sp = new SlicePredicate();
		sp.setColumn_names(colNames);

		// setup column parent
		ColumnParent cp = new ColumnParent("Standard3");

		this.iterateEntireColumnFamily(client, ks, cp, sp, "", "", 5);
	} finally {
		if (pool != null) {
			pool.releaseClient(client);
		}
	}

}

public static void main(String[] args) throws IllegalStateException, PoolExhaustedException, Exception {
	IterateExample dao = new IterateExample();
	dao.setupAndIterate();
}
}

I’m going to outline the Cassandra version of SELECT col_name_1 FROM table_name LIMIT 100

Step 1) Setup the connection pool and the client

CassandraClientPool pool = CassandraClientPoolFactory.INSTANCE.get();
CassandraClient client = pool.borrowClient("localhost", 9160);

Step 2) Define which Keyspace you want to use. This is equivalent to which DB or Schema.

Keyspace keyspace = client.getKeyspace("Keyspace1");

Step 3) Setup the slice predicate by defining which columns within the row you want returned. This is equivalent to SELECT col_name_1

// just look for one column for now
List<byte[]> colNames = new ArrayList<byte[]>();
colNames.add("column-name".getBytes());

// setup the slice predicate
SlicePredicate sp = new SlicePredicate();
sp.setColumn_names(colNames);

Step 4) Specify which Column Family to use. This is equivalent to a table or FROM table_name in SQL

// setup column parent (CF)
ColumnParent cp = new ColumnParent("Standard3");

Step 5) Execute the request. This is equivalent to executing a SQL query.

// get all the keys with a limit of 100 values returned
Map<String, List<Column>> map = keyspace.getRangeSlice(cp, sp, "", "", 100);

Note that the 3rd and 4th method args are empty strings. When you combine two empty strings for the start and end range, it is like saying SELECT col_name_1 FROM table_name without a WHERE clause.
Also if you have records that were deleted, the List of Column objects will be empty but the key will still be returned until is GC’d by Cassandra.

This example was used against a single Cassandra node using the Random Partitioner. Results will be returned to you in an unordered fashion.

In a single node configuration by default, Cassandra will not immediately fsync() your write to the commit log from buffer cache to disk. I discovered this (thanks to Jon via the users mailing list) when I was trying to do an insert and immediately tried to shutdown Cassandra and start it again. What I found was if I did that within a second or two, Cassandra’s commit log did not have my insert. If I waited around 5-10 seconds before shutting it down it did. So if you see this behavior this is what is happening. Now, a single node configuration for production is not desirable but I think it’s worth blogging about because it’s a common configuration for someone who is just hacking around and evaluating Cassandra for the first time.

Cassandra will let you achieve guaranteed durability for the commit log if you want. You can change this by changing “CommitLogSync” from periodic to batch and setting “CommitLogSyncBatchWindowInMS” to an appropriate number. This will obviously come with performance trade offs but at least you get to pick what is right for you. Also, if you are running a multi node configuration this may not be needed, assuming you are doing replication for your writes as Cassandra won’t acknowledge the write until that replication has finished. By doing this you virtually eliminate the effects of a hard system crash and can relax the durability of commit log writes for individual nodes.

Cassandra Durability Wiki: http://wiki.apache.org/cassandra/Durability

The goal here is to figure out using the top command in linux combined with thread dumps to figure out which thread and java code is consuming CPU resources. Below is a step by step walk through on how to accomplish this.

Step 1

Execute top command to determine the process id (PID) for the JBoss server. It will be listed under “java”

Step 2

Run the top command to show only the threads associated with the java process.
Execute top –H –p PID
(In this example the PID is 10937)

Step 3

Execute kill -3
to obtain a stack trace (In this example the PID is 10937)
Note: You can also utilize the dump stack capability in the JBoss JMX Console under the ServerInfo MBean

Step 4

Take the PID of the thread you want to correlate and convert it to hex using calc on Windows.

Step 5

Open the log file and do a search for the hex representation (native ID) of the thread ID. In this example it will be “2ae1”

This effectively tells you the line of code that is responsible for consuming the CPU resources.

The beauty of this “new” kind of e-commerce is that it’s already been done…well at least on TV. I mean let’s not forget about how easy it is to get sucked into QVC at 1 in the morning after you forgot to pay your expanded cable bill. All kidding aside, what I love best is the minimalist approach these sites have taken on. It is in fact, a common trend in most Web 2.0 sites and encourages a specific set of workflows for the user to follow. And when it comes to retailing, you want to make it is as simple as possible for someone to open their wallet. Amazon is definitely an early pioneer in this area with their fancy one click purchasing.

With this trend proving itself, it seems to me that the giant retailers of the world ought to pay attention to this racket. I think it’s inevitable that the Target’s, Wal-Mart’s, etc, will get into this independently or through some sort of collaborative network. If retailers don’t jump on this first, I can easily see other industries utilizing this as a way of getting in front of consumers without overwhelming them.

It’s certainly easy as a software engineer to just believe the DB is this magical box that self optimizes and all you have to do is turn on various settings to make it go faster. I wanted to share a real word example of how that line of thinking can get you into hot water, quickly.

The Situation

We have a MySQL 5 database in our hosted environment which runs on a dedicated dual quad core with 4 gigs of RAM. The only reason why I highlight the hardware here is because only a programmer could bring that type of computing power to its knees .

The table in question had about 850k rows in it, rapidly approaching 1M. In my experience, once you start approaching this level, at least with MySQL, you better have a really good idea how that table is being queried. In this case, we realized we were doing more writes than reads and we had an index on the table that really wasn’t necessary after all from a cost benefit point of view. Also, the data in the table could be purged more frequently to reduce the number of rows at any given time which decreases the time needed to rebuild the index after each insert, update, and delete operation.

Below are some graphs of the performance impact:

Two Cliffs

Week Long View

Lesson Learned

Even though initially, we thought that indexing a hash value (yes I know its a String field) would benefit our lookups, it ended up hurting us more than anything. Everyone’s situation is a little different but my hope is that I outlined a couple options to alleviate specific performance bottlenecks. If for example, you don’t have the luxury of just purging rows, you might want to consider archiving records into another table. I won’t comment any further on this specific topic because I believe this ventures into data warehousing which I am no expert on.

You won’t find this darling on match.com but I can guarantee you the inside is just as beautiful as the outside. I came across JWebUnit after becoming completely frustrated with Canoo WebTest. My team and I were running into many cases where tests would fail and because of a bug that hasn’t been fixed for quite some time, it doesn’t save the HTML output, thus not providing you with any immediate insight to why a test failed. This forced us to either guess or spend about a half hour trying to repeat the test case just to find out what happened.

Needless to say, JWebUnit allows easy access to the current state of the HTML page whenever you want it. Additionally, it will keep the state of all forms so you can see exactly what the framework set for values.

Wrap me up

JWebUnit is in a round about way, a select set of wrappers that covers about 90% of all web testing operations for a typical application. Also, because it is Java and extends the JUnit framework, abstraction is a breeze.

We found on average, we were writing 30-40% less lines of code in JWebUnit vs Canoo WebTest. Now you can certainly argue this may be a little unfair seeing as Canoo is a declarative framework but perhaps that is just a flaw due to its implementation.

My team and I had to add some additional helper methods to make testing easier but it was simple and very straight forward with basic object oriented design methods.

AJAX Support

As with Canoo WebTest, a simple Thread.sleep() for a second or two will allow you to do assertions on ajax based forms.

Needed Improvements

I noticed a couple items this framework was missing that Canoo WebTest offered:

1)  I noticed early on in our conversion there was a lack of support for direct XPath. There is currently no way via JWebUnit directly to evaluate Boolean XPath expressions. I’m currently looking for a workaround on this and also noticed it’s a rather highly requested feature on SourceForge.

2) The ability to perform assertions on row and column composition was absent. To remedy this situation, I had to write my own wrapper method but it was fairly easy. See below:

**Note WordPress takes out the full entity reference for non breaking spaces, so I just denoted it as “nbsp”:

public void assertValueInCell(String tableID, int rowNum, int colNum, String value) {
        assertElementPresent(tableID);
        Table reqTable = getTable(tableID);
        String[][] tableLookup = new String[reqTable.getRowCount() + 1][15];
        int rowCount = 0;
        int colCount = 0;

        Iterator rowItr = reqTable.getRows().iterator();
        while (rowItr.hasNext()) {
            //reset column counter for every new row iterated over
            colCount = 0;

            Row row = rowItr.next();
            Iterator cells = row.getCells().iterator();

            while (cells.hasNext()) {
                Cell cell = cells.next();

                tableLookup[rowCount][colCount] = cell.getValue().replaceAll("nbsp", "");
                colCount++;
            }

            rowCount++;
        }

        String tableVal = tableLookup[rowNum][colNum];
        if (!tableVal.equals(value)) {
            fail("Value: "
                    + value
                    + " not found in cell: ["
                    + rowNum
                    + ","
                    + colNum
                    + "]  -- Found this instead: "
                    + tableVal);
        }
    }

Reconciling the professional growth needs of developers against business goals is one of the hardest things I’ve come across as a development manager. In a software company there is seemingly countless reactions for every action and customer needs coming from every angle. Making sure your developers are getting what they need career wise adds a twist to an already challenging set of responsibilities.

It is common to have this problem only examined once a year during an annual review. Generally, some assembly of a plan is drafted up containing your goals for the following year but often times those plans succumb to “We just need to get through this next release” type of mantra and are quickly forgotten.

When I first started noticing that this was an important and ethical responsibility for my post, I found it difficult because as a co-owner I wanted the highest financial success possible.  After a lot of thought and informal discussions with my team, it became quite obvious about how passionate each member was about specific tasks they have worked on in the past or wanted to get involved with. It’s hard to put a direct dollar value on employee happiness, but after one experiences different groups of people in different corporate cultures, you can either sense the energy or you can’t. The companies that have this energy have passionate people who are directly or indirectly increasing their skill set, adding value to their organization. An overstated example of this would be Google. Developers are given time to explore their creative capabilities and yield eye popping technologies like Google Maps.

Knowledge complacency

Assigning the right task to the right person makes a lot of sense….on paper. The task is worked on by an individual deemed most competent and in theory, the best artifacts are produced in the shortest amount of time. I’m a firm proponent that this cycle needs to be broken from time to time at the very least, if not on a regular basis.

At some point or another, team members will start to notice their peers advancing in their fields or perhaps working on the next cutting edge. Most developers realize at some level that when you become complacent with your skill set, you become less valuable. Software developers are problems solvers but only in a realm that is defined first by their own knowledge and capabilities followed by the environment in which they are placed in. It is advantageous to any technical organization to not only be willing but encourage completely new ideas to solve problems. This includes but is not limited to re-architecting with different technologies and methods. By adapting this culture, engineers feel investing their own time in learning about new technologies and design philosophies will directly benefit them and the organization in which they are employed. This type of culture however, can be difficult to justify and defend in the face “growth at all costs” mentality. In this context, it is nearly impossible to justify the short term benefits of being a pro research & development organization.

In the end, it is difficult to have a technical company firing on all cylinders without being cognitive of how to balance both cultures. Simply adapting one or the other in whole will likely lead to long term imbalance and/or a company with great financial difficulty.

Until next time…..

Knowing the precise amount of resources and time to focus on satisfying both needs is a skill I look forward to sharpening. I hope in the near future to share my observations and strategies on this subject.

This post will go over the basic points of creating a web form that uploads multiple files. There is some trickery here that I got tripped up on that I will make blatantly obvious here.

The Form

<script>
function addFile(id) {
    var parentElm  = document.getElementById(id);
    var file = document.createElement('input');
    file.setAttribute('name','upload');
    file.setAttribute('type','file');

    parentElm.appendChild(file);
}
</script>
<s:form action="uploadFileExample"  enctype="multipart/form-data">
	<div id="fileDiv">
		Atttach   <s:file name="upload" label="File"/>
	</div>
	<a href="javascript:addFile('fileDiv');">Attach another file</a>
	<s:submit/>
</s:form>

The most important part here is that you take notice of whatever you name the file input element denoted in BOLD.

The Action

private File[] file;
private String[] contentType;
private String[] filename;

public void setUpload(File[] file) {
	this.file = file;
}

public void setUploadContentType(String[] contentType) {
	this.contentType = contentType;
}

public void setUploadFileName(String[] filename) {
	this.filename = filename;
}

public String execute() throws Exception {
       //...
}

Notice the BOLD in the method signatures. The key here is whatever you put into the name attribute from the step above, must go into the method name after the set prefix and before the actual object reference suffix.

From here, you’ll have a a File[] that you can loop over. Keep in mind, Struts 2 will upload the files to a temporary directory, depending on what application server you use, this will vary. The best thing to do here is just loop over each file that was uploaded and move each file into a more permanent location. And of course the other arrays contain exactly what you may think they contain. For moving files from one location to another: Move File

This post will illustrate one of the many ways to utilize Struts 2 to deal with handling global exception. Additionally, this topic will also include a way to rollback transactions for Hibernate.

The game plan

1) Let’s extend the existing ExceptionMappingInterceptor and just override only what we need:

public class MyExceptionInterceptor extends ExceptionMappingInterceptor {

    private static final Logger log = Logger.getLogger(MyExceptionInterceptor.class);

    protected void publishException(ActionInvocation invocation, ExceptionHolder exceptionHolder) {
        try {
            log.error(exceptionHolder.getExceptionStack());

             HibUtil.rollback();
       } catch (Exception e) {
            log.error(e, e);
        }

        super.publishException(invocation, exceptionHolder);
    }
}

2) Now we need to define our new interceptor and redefine the default stack:

<interceptors>
		<interceptor name="exception" class="MyExceptionInterceptor" />

		<interceptor-stack name="defaultStack">
		<interceptor-ref name="exception" />
		<interceptor-ref name="alias" />
		<interceptor-ref name="servletConfig" />
		<interceptor-ref name="prepare" />
		<interceptor-ref name="i18n" />
		<interceptor-ref name="chain" />
		<interceptor-ref name="debugging" />
		<interceptor-ref name="profiling" />
		<interceptor-ref name="scopedModelDriven" />
	        <interceptor-ref name="modelDriven" />
		<interceptor-ref name="fileUpload" />
		<interceptor-ref name="checkbox" />
		<interceptor-ref name="staticParams" />
		<interceptor-ref name="params">
			<param name="excludeParams">dojo\..*</param>
		</interceptor-ref>
		<interceptor-ref name="conversionError" />
		<interceptor-ref name="validation">
			<param name="excludeMethods">input,back,cancel,browse</param>
		</interceptor-ref>
		<interceptor-ref name="workflow">
			<param name="excludeMethods">input,back,cancel,browse</param>
		</interceptor-ref>
        </interceptor-stack>
</interceptors>

<default-interceptor-ref name="defaultStack" />

3)  Define a global transaction handler:

<global-results>
	<result name="Exception">/jsp/error/exception.jsp</result>
</global-results>
<global-exception-mappings>
	<exception-mapping exception="java.lang.Exception" result="Exception" />
</global-exception-mappings>

4) The following properties will be present on the stack if you wish to write a  simple JSP to display the error:

<s:property value="%{exception.message}"/>
<s:property value="%{exceptionStack}"/>

Final Thoughts

One thing that I have either not figured out or just simply don’t like is the inability to just extend an interceptor definition. Struts 2 seems to force the developer to redefine an entire stack in order for the new interceptor to be used. I hope I’m not missing something fundamental here but during my tests I wasn’t able to get that working and found no documentation to support my initial assumptions.  It almost seems that this configuration framework only fufills a subset of object oriented behavior. In any case, that wraps up this episode. I hope this gives some basic insight on how to put global exception handling and transaction rollback management in to your application.