Java中加载大数据展示到前台

2017.12.23 java

1.需求1: 读取大数据在前端展示

方案:分页每次取1000条数据

需求2: 在不分页的情况,取出的数据可能有10W上,将数据按条件进行分组汇总,不同单位,根据单位相同的计算组合计(单位可能有多个的情况)

方案1: 1.每次最大取1000条数据,放入文件中 2.按照分组条件,每个文件自行汇总 实现之后,当数据达到达到几万之后,速度非常慢 算法: 1.前台生产唯一的请求查询标识pk,传给后台 2.根据唯一标识生成汇总描述文件pkSum(x).txt,存放内容如下 根据汇总条件获取到的唯一标识compisiteKey compositeKey|count|pages|amount|currency|unit|sum|unit|sum|…… 实现的时候,获取一条记录,从pkSum(x).txt顺序查找,是否已经记录过,如果没有,则追加一行记录,重新生成此文件, 然后把详细信息记录到pk(x).txt文件中,其中x是页数,超过1000就另外新建文件存储, 这种方式实现之后,几万条数据就非常慢,因为文本文件本身没有修改功能,只能转储

File file = null;
InputStreamReader isr = null;
BufferedReader br = null;
StringBuilder sb = new StringBuilder();
boolean founded = false;

file = new File(...);
if (file.exists()) {
	isr = new InputStreamReader(new FileInputStream(file));
	br = new BufferedReader(isr);
	String lineText = null;
	while ((lineText = br.readLine()) != null) {
		StringTokenizer st = new StringTokenizer(lineText, "|");
		while (st.hasMoreTokens()) {
			// compositeKey|count|pages|amount|currency|unit|sum|unit|sum|......
			String token = CommonParser.toString(st.nextToken());
			if (compositeKey.equals(token)) {
				// 找到了:计算汇总值 & 记录数增加1 & 计算总页数
				......
				founded = true;
			}
			sb.append(lineText);
			sb.append("\r\n");
		}
		if (founded) {
			// 先删除文件,再新建写入:因为文本不能编辑,只能新增			
			file = new File(...);
			if (file.delete()) {
				FileUtils.writeStringToFile(file, sb.toString());
				break;
			}
		}
	}
}

方案2: 使用Trie树建立倒排索引(RevertedIndex) 建立索引过程: 1.从数据库获取数据 2.先存到Trie树中,自动建立倒排索引,并汇总 3.Trie建好的索引保存到pk_idx.txt索引文件中,详细信息保存到pk(x).txt文件中 4.读取的时候,先顺序读取pk_idx.txt索引文件,从索引文件再找相应的详细信息文件进行针对性读取

索引文件:

格式:compositeKey count pages amount currency unit sum unit sum ……

compositeKey: 汇总条件 count: 记录数 pages: 总页数 amount: 金额合计 currency: 货币类型 unit: 单位 sum: 合计数量 …多个单位继续…

000008002496|4|1|1350398.43|CNY|[x1]|{T=7698, M3=5167680, DEG=410386.2, KG=2379730}
000008002497|4|1|1439320.99|CNY|[x1]|{T=7893, M3=5275120, DEG=760214.4, KG=2452580}

Trie.java

import java.math.BigDecimal;
import java.util.List;

public interface Trie {
	public void add(String item, String path, String unit, BigDecimal quantity, BigDecimal amount, String currency, int pageSize);	
	public List<Item> getItemsWithPrefix(String prefix);
	public Item getCount(String item);
	public List<Item> getAllItems();
	public List<String> getMultiItems();
	public List<String> getSingleItems();
}

Item.java

import java.math.BigDecimal;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.StringUtils;

public class Item {

	private int count = 0;
	private Set<String> paths = new HashSet<String>();
	private Map<Integer, Item> nextItems = new HashMap<Integer, Item>();
	private Map<String, BigDecimal> sumQty = new HashMap<String, BigDecimal>();
	private BigDecimal amount = BigDecimal.ZERO;
	private String currency;
	private int page = 1;
	private boolean isRead = false;

	public int getCount() {
		return count;
	}

	public void setCount(int count) {
		this.count = count;
	}

	public Set<String> getPaths() {
		return paths;
	}

	public void setPaths(Set<String> paths) {
		this.paths = paths;
	}

	public Map<Integer, Item> getNextItems() {
		return nextItems;
	}

	public void setNextItems(Map<Integer, Item> nextItems) {
		this.nextItems = nextItems;
	}

	public Map<String, BigDecimal> getSumQty() {
		return sumQty;
	}

	public void setSumQty(Map<String, BigDecimal> sumQty) {
		this.sumQty = sumQty;
	}

	public BigDecimal getAmount() {
		return amount;
	}

	public void setAmount(BigDecimal amount) {
		this.amount = amount;
	}

	public String getCurrency() {
		return currency;
	}

	public void setCurrency(String currency) {
		this.currency = currency;
	}

	public int getPage() {
		return page;
	}

	public void setPage(int page) {
		this.page = page;
	}

	public boolean isRead() {
		return isRead;
	}

	public void setRead(boolean isRead) {
		this.isRead = isRead;
	}

	@Override
	public String toString() {
		// compositeKey|count|pages|amount|currency|unit|sum|unit|sum|......
		return String.format(StringUtils.repeat("%s", "|", 6) + "\r\n", count, page, amount, currency, paths, sumQty);
	}
}

HashMapTrie.java

package com.eppen.sap;

import java.math.BigDecimal;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;

/**
 * 
 * @author fuyouming
 * 
 */
public class HashMapTrie implements Trie {

	private final Item root = new Item();

	public void add(String str, String path, String unit, BigDecimal quantity, BigDecimal amount, String currency, int pageSize) {
		add(root, str, path, unit, quantity, amount, currency, pageSize);
	}

	public List<Item> getItemsWithPrefix(String prefix) {
		List<Item> list = new LinkedList<Item>();
		Item current = walk2FixedItem(prefix);
		if (current != null) {
			walk2NextItem(list, current, new StringBuffer(prefix));
		}
		return list;
	}

	public Item getCount(String str) {
		Item current = walk2FixedItem(str);
		if (current != null) {
			return current;
		}
		return current;
	}

	public List<Item> getAllItems() {
		List<Item> list = new LinkedList<Item>();
		Iterator<Integer> iterator = root.getNextItems().keySet().iterator();
		while (iterator.hasNext()) {
			Integer i = iterator.next();
			Item current = (Item) root.getNextItems().get(i);
			StringBuffer buf = new StringBuffer().append((char) i.intValue());
			walk2NextItem(list, current, buf);
		}
		return list;
	}

	public List<String> getMultiItems() {
		return null;
	}

	public List<String> getSingleItems() {
		return null;
	}

	private void add(Item item, String str, String path, String unit, BigDecimal quantity, BigDecimal amount, String currency, int pageSize) {
		if (str == null || str.length() == 0) {
			item.setCount(item.getCount() + 1);
			item.getPaths().add(path);
			
			item.setAmount(item.getAmount().add(amount == null ? BigDecimal.ZERO : amount));
			item.setCurrency(currency);
			
			item.getSumQty().put(unit, item.getSumQty().get(unit) == null ? quantity : item.getSumQty().get(unit).add(quantity));
			
			if (item.getCount() % pageSize == 0) {
				item.setPage(item.getPage() + 1);
			}
		} else {
			int index = str.charAt(0);
			Item nextItem = null;
			if (!item.getNextItems().containsKey(index)) {
				nextItem = new Item();
				item.getNextItems().put(index, nextItem);
			} else {
				nextItem = (Item) item.getNextItems().get(index);
			}
			add(nextItem, str.substring(1), path, unit, quantity, amount, currency, pageSize);
		}
	}

	private Item walk2FixedItem(String prefix) {
		if (prefix.length() > 0) {
			Item current = root;
			for (int i = 0; i < prefix.length(); i++) {
				int index = prefix.charAt(i);
				current = current.getNextItems().get(index);
				if (current == null) {
					break;
				}
			}
			return current;
		} else {
			return null;
		}
	}

	private void walk2NextItem(List<Item> list, Item previous, StringBuffer buf) {
		for (int i = 0; i < previous.getCount(); i++) {
			list.add(previous);
		}
		Iterator<Integer> iterator = previous.getNextItems().keySet().iterator();
		while (iterator.hasNext()) {
			Integer i = iterator.next();
			Item current = (Item) previous.getNextItems().get(i);
			StringBuffer newBuf = new StringBuffer(buf).append((char) i.intValue());
			walk2NextItem(list, current, newBuf);
		}
	}

	public static void main(String args[]) {
		Trie trie = new HashMapTrie();
		trie.add("ab", "C:/x1", "KG", new BigDecimal("5.5"), new BigDecimal("16.88"), "CNY", 1000);
		trie.add("abc", "C:/x2", "KG", new BigDecimal("6.5"), new BigDecimal("26.88"), "CNY", 1000);
		trie.add("abcd", "C:/x3", "KG", new BigDecimal("7.5"), new BigDecimal("36.88"), "CNY", 1000);
		trie.add("abd", "C:/x4", "KG", new BigDecimal("8.5"), new BigDecimal("46.88"), "CNY", 1000);
		trie.add("abd", "C:/x5", "KG", new BigDecimal("9.5"), new BigDecimal("56.88"), "CNY", 1000);
		trie.add("a cbd", "C:/x6", "KG", new BigDecimal("15.5"), new BigDecimal("66.88"), "CNY", 1000);
		trie.add("acrtfg", "C:/x7", "KG", new BigDecimal("25.5"), new BigDecimal("76.88"), "CNY", 1000);

		List<Item> list = trie.getAllItems();
		for (Item item : list) {
			System.out.println(item);
		}
		System.out.println("=================================");
		list = trie.getItemsWithPrefix("ac");
		for (Item item : list) {
			System.out.println(item);
		}
	}
}

更新列表：

参考文章：

[1]: [2]: [3]: [4]:

相关阅读