纯真IP地址数据库qqwry.dat解析
2017-07-16 本文已影响2936人
迪峰
ip地址数据库，在现在互联网时代非常有用，比如大型网站的用户安全保护系统，就常常会根据ip反查的信息，甄别账号的一些不安全登录行为，比如跨区域登录问题等。ip其实关联了一些有信息，比如区域，所在运营商，一些收录全的，甚至包括具体经纬度，像百度的IP定位api就比较全。下面来介绍一下“ 纯真IP地址数据库qqwry”的格式以及解析
以下是“ 纯真IP地址数据库qqwry”官网对其的介绍。
纯真版IP地址数据库是当前网络上最权威、地址最精确、IP记录以及网吧数据最多的IP地址数据库。收集了包括中国电信、中国移动、中国联通、铁通、长城宽带等各 ISP 的最新准确 IP 地址数据。通过大家的共同努力打造一个没有未知数据，没有错误数据的QQ IP。IP数据库每5天更新一次，请大家定期更新最新的IP数据库！
格式
+———-+
| 文件头 | (8字节)
+———-+
| 记录区 | (不定长)
+———-+
| 索引区 | (大小由文件头决定)
+———-+
使用java语言解析的两种思路：
-
使用内存映射文件方式读取，使用java的MappedByteBuffer 将原数据文件映射到MappedByteBuffer对象中，然后通过MappedByteBuffer 提供的字节读取方式实现ip的查找。搜索是在索引区使用二分法
-
使用byte数组读取，及将二进制的数据库信息全都按顺序读入到一个数组中，由于数据是有格式的，我们便可计算根据索引区和记录区在数组中的位置，当查询ip时，从数组中的索引区开始通过二分查找方式找到IP地址对应的国家和区域的位置，然后从数组中取出地区信息。
热升级思路：
使用一个可调度的单线程的线程池，线程定时检测qqwry.dat文件是否修改，若修改则重新将数据重新载入，载入过程可使用可重入锁ReentrantLock来锁住资源，避免在更新的过程中脏查询
两种解析方式的实现源码如下：
方式一（MappedByteBuffer ）：
package com.difeng.qqwry1;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/**
* @Description:ip定位查找工具(使用内存映射文件方式读取，线程安全)
* @author:difeng
* @date:2016年12月11日
*/
public class IPLocation {
private static final int IP_RECORD_LENGTH = 7;
private static final byte REDIRECT_MODE_1 = 0x01;
private static final byte REDIRECT_MODE_2 = 0x02;
private MappedByteBuffer mbbFile;
private static Long lastModifyTime = 0L;
public static boolean enableFileWatch = false;
private static ReentrantLock lock = new ReentrantLock();
private File qqwryFile;
private long firstIndexOffset;
private long lastIndexOffset;
private long totalIndexCount;
public IPLocation(String filePath) throws Exception {
this.qqwryFile = new File(filePath);
load();
if (enableFileWatch) {
watch();
}
}
private void watch(){
Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
long time = qqwryFile.lastModified();
if (time > lastModifyTime) {
lastModifyTime = time;
try {
load();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}, 1000L, 30000L, TimeUnit.MILLISECONDS);
}
public long read4ByteAsLong(long pos) {
mbbFile.position((int)pos);
return 0xFFFFFFFFL & mbbFile.getInt();
}
public long read3ByteAsLong(long pos){
mbbFile.position((int)pos);
return 0xFFFFFFL & mbbFile.getInt();
}
@SuppressWarnings("resource")
private void load() throws Exception {
lastModifyTime = qqwryFile.lastModified();
lock.lock();
try {
mbbFile = new RandomAccessFile(qqwryFile, "r")
.getChannel()
.map(FileChannel.MapMode.READ_ONLY, 0, qqwryFile.length());
mbbFile.order(ByteOrder.LITTLE_ENDIAN);
firstIndexOffset = read4ByteAsLong(0);
lastIndexOffset = read4ByteAsLong(4);
totalIndexCount = (lastIndexOffset - firstIndexOffset) / IP_RECORD_LENGTH + 1;
} finally {
lock.unlock();
}
}
/**
* @Description:将“.”号分隔的字符串转换为long类型的数字，字节序例如:
* ip:182.92.240.48 16进制表示(B6.5C.F0.30)
* 转换后ip的16进制表示:0xB65CF030
* @param ipStr
* @return:long
*/
private static long inet_pton(String ipStr) {
if(ipStr == null){
throw new NullPointerException("ip不能为空");
}
String [] arr = ipStr.split("\\.");
long ip = (Long.parseLong(arr[0]) & 0xFFL) << 24 & 0xFF000000L;
ip |= (Long.parseLong(arr[1]) & 0xFFL) << 16 & 0xFF0000L;
ip |= (Long.parseLong(arr[2]) & 0xFFL) << 8 & 0xFF00L;
ip |= (Long.parseLong(arr[3]) & 0xFFL);
return ip;
}
private long search(long ip) {
long low = 0;
long high = totalIndexCount;
long mid = 0;
while(low <= high) {
mid = (low + high) >>> 1 ;
long indexIP = read4ByteAsLong(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH);
long nextIndexIP = read4ByteAsLong(firstIndexOffset + mid * IP_RECORD_LENGTH);
if(indexIP <= ip && ip < nextIndexIP) {
return read3ByteAsLong(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH + 4);
} else {
if(ip > indexIP) {
low = mid + 1;
} else if(ip < indexIP) {
high = mid - 1;
}
}
}
return -1;
}
private Location readIPLocation(long offset) {
try {
mbbFile.position((int)offset + 4);
Location loc = new Location();
byte redirectMode = mbbFile.get();
if (redirectMode == REDIRECT_MODE_1) {
long countryOffset = read3ByteAsLong((int)offset + 5);
mbbFile.position((int)countryOffset);
redirectMode = mbbFile.get();
if (redirectMode == REDIRECT_MODE_2) {
loc.country = readString(read3ByteAsLong(countryOffset + 1));
mbbFile.position((int)countryOffset + 4);
} else {
loc.country = readString(countryOffset);
}
loc.area = readArea(mbbFile.position());
} else if (redirectMode == REDIRECT_MODE_2) {
loc.country = readString(read3ByteAsLong((int)offset + 5));
loc.area = readArea((int)offset + 8);
} else {
loc.country = readString(mbbFile.position() - 1);
loc.area = readArea(mbbFile.position());
}
return loc;
} catch (Exception e) {
return null;
}
}
private String readArea(int offset) {
mbbFile.position(offset);
byte redirectMode = mbbFile.get();
if (redirectMode == REDIRECT_MODE_1 || redirectMode == REDIRECT_MODE_2) {
long areaOffset = read3ByteAsLong((int)offset + 1);
if (areaOffset == 0){
return "";
} else {
return readString(areaOffset);
}
} else {
return readString(offset);
}
}
private String readString(long offset) {
try {
mbbFile.position((int)offset);
byte[] buf = new byte[128];
int i;
for (i = 0, buf[i] = mbbFile.get(); buf[i] != 0; buf[++i] = mbbFile.get());
if (i != 0){
return new String(buf, 0, i, "GBK");
}
} catch (IOException e) {
e.printStackTrace();
}
return "";
}
public Location fetchIPLocation(String ip) {
lock.lock();
try {
long offset = search(inet_pton(ip));
if(offset != -1){
return readIPLocation(offset);
}
} finally {
lock.unlock();
}
return null;
}
}
方式二（数组方式）：
package com.difeng.qqwry2;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/**
* @Description:ip定位(使用byte数据方式读取)
* @author:difeng
* @date:2016年12月13日
*/
public class IPLocation {
private byte[] data;
private long firstIndexOffset;
private long lastIndexOffset;
private long totalIndexCount;
private static final byte REDIRECT_MODE_1 = 0x01;
private static final byte REDIRECT_MODE_2 = 0x02;
static final long IP_RECORD_LENGTH = 7;
private static ReentrantLock lock = new ReentrantLock();
private static Long lastModifyTime = 0L;
public static boolean enableFileWatch = false;
private File qqwryFile;
public IPLocation(String filePath) throws Exception {
this.qqwryFile = new File(filePath);
load();
if(enableFileWatch){
watch();
}
}
private void watch() {
Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
long time = qqwryFile.lastModified();
if (time > lastModifyTime) {
lastModifyTime = time;
try {
load();
System.out.println("reload");
} catch (Exception e) {
e.printStackTrace();
}
}
}
}, 1000L, 5000L, TimeUnit.MILLISECONDS);
}
private void load() throws Exception {
lastModifyTime = qqwryFile.lastModified();
ByteArrayOutputStream out = null;
FileInputStream in = null;
lock.lock();
try {
out = new ByteArrayOutputStream();
byte[] b = new byte[1024];
in = new FileInputStream(qqwryFile);
while(in.read(b) != -1){
out.write(b);
}
data = out.toByteArray();
firstIndexOffset = read4ByteAsLong(0);
lastIndexOffset = read4ByteAsLong(4);
totalIndexCount = (lastIndexOffset - firstIndexOffset) / IP_RECORD_LENGTH + 1;
in.close();
out.close();
} finally {
try {
if(out != null) {
out.close();
}
if(in != null) {
in.close();
}
} catch (IOException e) {
e.printStackTrace();
}
lock.unlock();
}
}
private long read4ByteAsLong(final int offset) {
long val = data[offset] & 0xFF;
val |= (data[offset + 1] << 8L) & 0xFF00L;
val |= (data[offset + 2] << 16L) & 0xFF0000L;
val |= (data[offset + 3] << 24L) & 0xFF000000L;
return val;
}
private long read3ByteAsLong(final int offset) {
long val = data[offset] & 0xFF;
val |= (data[offset + 1] << 8) & 0xFF00;
val |= (data[offset + 2] << 16) & 0xFF0000;
return val;
}
private long search(long ip) {
long low = 0;
long high = totalIndexCount;
long mid = 0;
while(low <= high){
mid = (low + high) >>> 1 ;
long indexIP = read4ByteAsLong((int)(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH));
long indexIPNext = read4ByteAsLong((int)(firstIndexOffset + mid * IP_RECORD_LENGTH));
if(indexIP <= ip && ip < indexIPNext) {
return read3ByteAsLong((int)(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH + 4));
} else {
if(ip > indexIP) {
low = mid + 1;
} else if (ip < indexIP) {
high = mid - 1;
}
}
}
return -1;
}
public Location fetchIPLocation(String ip) {
long numericIp = inet_pton(ip);
lock.lock();
long offset = search(numericIp);
try{
if(offset != -1) {
return readIPLocation((int)offset);
}
} finally {
lock.unlock();
}
return null;
}
private Location readIPLocation(final int offset) {
final Location loc = new Location();
try {
byte redirectMode = data[offset + 4];
if (redirectMode == REDIRECT_MODE_1) {
long countryOffset = read3ByteAsLong((int)offset + 5);
redirectMode = data[(int)countryOffset];
if (redirectMode == REDIRECT_MODE_2) {
final QQwryString country = readString((int)read3ByteAsLong((int)countryOffset + 1));
loc.country = country.string;
countryOffset = countryOffset + 4;
} else {
final QQwryString country = readString((int)countryOffset);
loc.country = country.string;
countryOffset += country.byteCountWithEnd;
}
loc.area = readArea((int)countryOffset);
} else if (redirectMode == REDIRECT_MODE_2) {
loc.country = readString((int)read3ByteAsLong((int)offset + 5)).string;
loc.area = readArea((int)offset + 8);
} else {
final QQwryString country = readString((int)offset + 4);
loc.country = country.string;
loc.area = readArea((int)offset + 4 + country.byteCountWithEnd);
}
return loc;
} catch (Exception e) {
return null;
}
}
private String readArea(final int offset) {
byte redirectMode = data[offset];
if (redirectMode == REDIRECT_MODE_1 || redirectMode == REDIRECT_MODE_2) {
long areaOffset = read3ByteAsLong((int)offset + 1);
if (areaOffset == 0) {
return "";
} else {
return readString((int)areaOffset).string;
}
} else {
return readString(offset).string;
}
}
private QQwryString readString(int offset) {
int pos = offset;
final byte[] b = new byte[128];
int i;
for (i = 0, b[i] = data[pos++]; b[i] != 0; b[++i] = data[pos++]);
try{
return new QQwryString(new String(b,0,i,"GBK"),i + 1);
} catch(UnsupportedEncodingException e) {
return new QQwryString("",0);
}
}
/**
* @Description:“.”号分隔的字符串转换为long类型的数字
* @param ipStr
* @return:long
*/
private static long inet_pton(String ipStr) {
if(ipStr == null){
throw new NullPointerException("ip不能为空");
}
String [] arr = ipStr.split("\\.");
long ip = (Long.parseLong(arr[0]) & 0xFFL) << 24 & 0xFF000000L;
ip |= (Long.parseLong(arr[1]) & 0xFFL) << 16 & 0xFF0000L;
ip |= (Long.parseLong(arr[2]) & 0xFFL) << 8 & 0xFF00L;
ip |= (Long.parseLong(arr[3]) & 0xFFL);
return ip;
}
private class QQwryString{
public final String string;
public final int byteCountWithEnd;
public QQwryString(final String string,final int byteCountWithEnd) {
this.string = string;
this.byteCountWithEnd = byteCountWithEnd;
}
@Override
public String toString() {
return string;
}
}
}
以上为主要代码，获取全部代码请点击全部代码
使用
final IPLocation ipLocation = new IPLocation(filePath);
Location loc = ipl.fetchIPLocation("182.92.240.50");
System.out.printf("%s %s",loc.country,loc.area);
格式改进
由于原格式中读取地区记录时采用重定向，有些繁琐。去掉之后格式更简单，国家和地区单独存放,索引里分别记录的国家和地区的地址。
新格式如下：
+----------+
| 文件头 | (8字节)
+----------+
| 记录区 | (不定长)
+----------+
| 索引区 | (大小由文件头决定)
+----------+
文件头：
+------------------------------+-----------------------------+
| first index position(4 bytes)|last index position(4 bytes) |
+------------------------------+-----------------------------+
记录区：
+------------------+----------+------------------+----------+-----
| country1(n bytes)|\0(1 byte)| country2(n bytes)|\0(1 byte)|...
+------------------+----------+------------------+----------+-----
+------------------+----------+------------------+----------+-----
| area1(n bytes) |\0(1 byte)| area2(n bytes) |\0(1 byte)|...
+------------------+----------+------------------+----------+-----
索引区：
+------------+-------------------------+------------------------+
|ip1(4 bytes)|country position(3 bytes)| area position(3 bytes) |...
+------------+-------------------------+------------------------+
转换方法：
final IPFileConvertor convertor = new
IPFileConvertor(IPFileConvertor.class.getResource("/qqwry.dat").getPath(),"./qqwry.dat");
convertor.convert();
新格式使用方法和之前的一致，使用com.difeng.convert包下的解析类IPLocation解析即可。
相关连接：
qqwry下载： qqwry
全球ip地址库（收费）：IPLocation
