新增地级市成交价抓取

src/main/java/com/leeroa/dydb/datasource/lianjia/bo/DataLianjiaBo.java

@@ -36,6 +36,19 @@ public class DataLianjiaBo implements BO {
     @Condition(matchMode = MatchModel.OR, targets = {"%houses", "%address"})
     private String keywords;
 
+    @ApiField(value = "城市码")
+    @Condition
+    private String cityCode;
+
+
+    public String getCityCode() {
+        return cityCode;
+    }
+
+    public void setCityCode(String cityCode) {
+        this.cityCode = cityCode;
+    }
+
     public String getBizName() {
         return bizName;
     }

src/main/java/com/leeroa/dydb/datasource/lianjia/bo/LianjiaParams.java

@@ -8,6 +8,7 @@ import java.util.Date;
  * @author michael
  */
 public class LianjiaParams {
+
     @ApiField("城市编码")
     private String cityCode;
     private String areaName;
@@ -137,4 +138,5 @@ public class LianjiaParams {
     public void setPage(Integer page) {
         this.page = page;
     }
+
 }

src/main/java/com/leeroa/dydb/datasource/lianjia/cache/LianjiaCache.java

@@ -1,5 +1,6 @@
 package com.leeroa.dydb.datasource.lianjia.cache;
 
+import cn.hutool.core.util.StrUtil;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.leeroa.dydb.datasource.paimai.utils.LianjiaUtils;
@@ -20,10 +21,13 @@ public class LianjiaCache {
             .build();
 
 
-    public static Map<String, String> get(String code) {
+    public static Map<String, String> get(String code,String cityCode) {
         Map<String, String> bizList = cache.getIfPresent(code);
         if (bizList == null || bizList.keySet().size() == 0) {
-            bizList = LianjiaUtils.fetchBizByAreaCode(code);
+            if (StrUtil.isBlank(cityCode)){
+                cityCode = "cd";
+            }
+            bizList = LianjiaUtils.fetchBizByAreaCode(code,cityCode);
             if (bizList != null) {
                 cache.put(code, bizList);
             }
@@ -31,12 +35,12 @@ public class LianjiaCache {
         return bizList;
     }
 
-    public static List<String> getBizCodeList(String code) {
-        return new ArrayList<>(get(code).keySet());
+    public static List<String> getBizCodeList(String code,String cityCode) {
+        return new ArrayList<>(get(code,cityCode).keySet());
     }
 
-    public static List<String> getBizNameList(String code) {
-        return new ArrayList<>(get(code).values());
+    public static List<String> getBizNameList(String code,String cityCode) {
+        return new ArrayList<>(get(code,cityCode).values());
     }
 
     public static void put(String code, Map<String, String> newMap) {

src/main/java/com/leeroa/dydb/datasource/lianjia/dao/DataLianjiaDao.java

@@ -2,6 +2,7 @@ package com.leeroa.dydb.datasource.lianjia.dao;
 
 import com.leeroa.dydb.datasource.lianjia.bo.DataLianjiaBo;
 import com.leeroa.dydb.datasource.lianjia.domain.DataLianjia;
+import com.leeroa.dydb.datasource.lianjia.domain.DataLianjiaCode;
 
 import java.util.List;
 
@@ -39,4 +40,6 @@ public interface DataLianjiaDao {
      */
     void delete(DataLianjia dataLianjia);
 
+
+
 }

src/main/java/com/leeroa/dydb/datasource/lianjia/domain/DataLianjia.java

@@ -29,6 +29,9 @@ import java.util.Date;
 })
 public class DataLianjia extends Location implements AttachmentSymbol {
 
+    @ApiField(value = "城市码")
+    @Column
+    private String cityCode;
     @ApiField(value = "链家的房屋ID")
     @Column(length = 20)
     private String houseNO;

src/main/java/com/leeroa/dydb/datasource/lianjia/service/DataLianjiaService.java

@@ -3,11 +3,13 @@ package com.leeroa.dydb.datasource.lianjia.service;
 import com.leeroa.dydb.datasource.lianjia.bo.DataLianjiaBo;
 import com.leeroa.dydb.datasource.lianjia.bo.LianjiaParams;
 import com.leeroa.dydb.datasource.lianjia.domain.DataLianjia;
+import com.leeroa.dydb.datasource.lianjia.domain.DataLianjiaCode;
 import com.leeroa.dydb.datasource.lianjia.vo.DataLianjiaVo;
 import com.michael.core.pager.PageVo;
 
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author Michael
@@ -115,4 +117,5 @@ public interface DataLianjiaService {
      */
     void cache(String areaCode, String bizCode, Date startDate, Date endDate);
 
+
 }

src/main/java/com/leeroa/dydb/datasource/lianjia/service/impl/DataLianjiaServiceImpl.java

@@ -9,9 +9,12 @@ import com.leeroa.dydb.datasource.HouseCache;
 import com.leeroa.dydb.datasource.lianjia.bo.DataLianjiaBo;
 import com.leeroa.dydb.datasource.lianjia.bo.LianjiaParams;
 import com.leeroa.dydb.datasource.lianjia.cache.LianjiaCache;
+import com.leeroa.dydb.datasource.lianjia.dao.DataLianjiaCodeDao;
 import com.leeroa.dydb.datasource.lianjia.dao.DataLianjiaDao;
 import com.leeroa.dydb.datasource.lianjia.domain.DataLianjia;
+import com.leeroa.dydb.datasource.lianjia.domain.DataLianjiaCode;
 import com.leeroa.dydb.datasource.lianjia.dto.DataLianjiaDTO;
+import com.leeroa.dydb.datasource.lianjia.service.DataLianjiaCodeService;
 import com.leeroa.dydb.datasource.lianjia.service.DataLianjiaService;
 import com.leeroa.dydb.datasource.lianjia.vo.DataLianjiaVo;
 import com.leeroa.dydb.datasource.paimai.utils.LianjiaResponse;
@@ -63,6 +66,9 @@ public class DataLianjiaServiceImpl implements DataLianjiaService, BeanWrapCallb
     @Resource
     private DataLianjiaDao dataLianjiaDao;
 
+    @Resource
+    private DataLianjiaCodeDao dataLianjiaCodeDao;
+
 
     @Override
     public String save(DataLianjia dataLianjia) {
@@ -209,7 +215,8 @@ public class DataLianjiaServiceImpl implements DataLianjiaService, BeanWrapCallb
             records++;
             d.setBatchNO(batchNO);
             d.setProvinceName("四川");
-            d.setCityName("成都");
+            d.setCityName(dataLianjiaCodeDao.getCityNameByCode(params.getCityCode()));
+            d.setCityCode(params.getCityCode());
             if (StringUtils.isEmpty(d.getAreaName())) {
                 d.setAreaName(params.getAreaName());
             }
@@ -393,7 +400,7 @@ public class DataLianjiaServiceImpl implements DataLianjiaService, BeanWrapCallb
             }
         });
         if (StringUtils.isNotEmpty(bizCode)) {
-            LianjiaCache.get(areaCode).forEach((k, v) -> {
+            LianjiaCache.get(areaCode,null).forEach((k, v) -> {
                 if (k.equals(bizCode)) {
                     criteria.add(Restrictions.eq("bizName", v));
                 }
@@ -405,6 +412,8 @@ public class DataLianjiaServiceImpl implements DataLianjiaService, BeanWrapCallb
         HouseCache.set(code, data);
     }
 
+
+
     @Override
     public void doCallback(DataLianjia dataLianjia, DataLianjiaVo vo) {
         ParameterContainer container = ParameterContainer.getInstance();

src/main/java/com/leeroa/dydb/datasource/lianjia/web/DataLianjiaCtrl.java

@@ -7,9 +7,12 @@ import com.google.gson.JsonObject;
 import com.leeroa.base.utils.DownloadResponseWrapper;
 import com.leeroa.base.utils.export.ExcelUtils;
 import com.leeroa.dydb.datasource.lianjia.bo.DataLianjiaBo;
+import com.leeroa.dydb.datasource.lianjia.bo.DataLianjiaCodeBo;
 import com.leeroa.dydb.datasource.lianjia.bo.LianjiaParams;
 import com.leeroa.dydb.datasource.lianjia.cache.LianjiaCache;
 import com.leeroa.dydb.datasource.lianjia.domain.DataLianjia;
+import com.leeroa.dydb.datasource.lianjia.domain.DataLianjiaCode;
+import com.leeroa.dydb.datasource.lianjia.service.DataLianjiaCodeService;
 import com.leeroa.dydb.datasource.lianjia.service.DataLianjiaService;
 import com.leeroa.dydb.datasource.lianjia.vo.DataLianjiaVo;
 import com.leeroa.dydb.datasource.paimai.utils.LianjiaUtils;
@@ -47,6 +50,9 @@ public class DataLianjiaCtrl extends BaseController {
     @Resource
     private DataLianjiaService dataLianjiaService;
 
+    @Resource
+    private DataLianjiaCodeService dataLianjiaCodeService;
+
 
     @RequestMapping(value = {""}, method = RequestMethod.GET)
     public String toList() {
@@ -184,18 +190,28 @@ public class DataLianjiaCtrl extends BaseController {
         GsonUtils.printSuccess(response);
     }
 
+    // 获取链家网四川城市列表
+    @RequestMapping("/urbanMap")
+    @ResponseBody
+    public void queryUrbanMap(HttpServletRequest request, HttpServletResponse response) {
+        //GsonUtils.printData(response, LianjiaUtils.getUrbanMap());
+    }
+
     // 获取成都的所有区域信息
     @RequestMapping("/citymap")
     @ResponseBody
     public void queryCityMap(HttpServletRequest request, HttpServletResponse response) {
-        GsonUtils.printData(response, LianjiaUtils.getCityMap());
+        //GsonUtils.printData(response, LianjiaUtils.getCityMap());
+        DataLianjiaCodeBo bo = GsonUtils.wrapDataToEntity(request, DataLianjiaCodeBo.class);
+        List<DataLianjiaCode> cityMap = dataLianjiaCodeService.queryCityMap(bo);
+        GsonUtils.printData(response, cityMap);
     }
 
     // 获取区域下的商圈信息
-    @RequestMapping(value = "/citymap-biz", params = "code")
+    @RequestMapping(value = "/citymap-biz")
     @ResponseBody
-    public void queryBizList(String code, HttpServletRequest request, HttpServletResponse response) {
-        Map<String, String> data = LianjiaCache.get(code);
+    public void queryBizList(String code,String cityCode, HttpServletRequest request, HttpServletResponse response) {
+        Map<String, String> data = LianjiaCache.get(code,cityCode);
         GsonUtils.printData(response, data);
     }
 

src/main/java/com/leeroa/dydb/datasource/lianjia/web/DataLianjiaUpCtrl.java

@@ -174,7 +174,7 @@ public class DataLianjiaUpCtrl extends BaseController {
     @RequestMapping(value = "/citymap-biz", params = "code")
     @ResponseBody
     public void queryBizList(String code, HttpServletRequest request, HttpServletResponse response) {
-        Map<String, String> data = LianjiaCache.get(code);
+        Map<String, String> data = LianjiaCache.get(code,null);
         GsonUtils.printData(response, data);
     }
 

src/main/java/com/leeroa/dydb/datasource/paimai/utils/LianjiaUpUtils.java

@@ -334,7 +334,7 @@ public class LianjiaUpUtils {
 
                 //  设置商圈
                 if (StringUtils.isNotEmpty(bzCode) && StringUtils.isNotEmpty(areaCode)) {
-                    Map<String, String> bzMap = LianjiaCache.get(areaCode);
+                    Map<String, String> bzMap = LianjiaCache.get(areaCode,null);
                     dataLianjiaUp.setBizName(bzMap.get(bzCode));
                 }
             }

src/main/java/com/leeroa/dydb/datasource/paimai/utils/LianjiaUtils.java

@@ -38,6 +38,7 @@ import java.util.regex.Pattern;
  */
 @Slf4j
 public class LianjiaUtils {
+
     /**
      * 成都市的行政区编码
      */
@@ -120,7 +121,8 @@ public class LianjiaUtils {
 
     public static LianjiaResponse fetch(LianjiaParams params) {
         // 查询成交数据
-        StringBuilder url = new StringBuilder("https://cd.lianjia.com/chengjiao/");
+        String cityCode = params.getCityCode();
+        StringBuilder url = new StringBuilder("https://"+cityCode+".lianjia.com/chengjiao/");
 
         Logger logger = Logger.getLogger(LianjiaUtils.class);
         // 商圈和行政区域同时只能存在一个，因为规则是一样的
@@ -458,8 +460,8 @@ public class LianjiaUtils {
         return x;
     }
 
-    public static Map<String, String> fetchBizByAreaCode(String areaCode) {
-        String url = "https://cd.lianjia.com/ershoufang/" + areaCode + "/";
+    public static Map<String, String> fetchBizByAreaCode(String areaCode,String cityCode) {
+        String url = "https://"+cityCode+".lianjia.com/ershoufang/" + areaCode + "/";
         Logger logger = Logger.getLogger(LianjiaUtils.class);
         logger.info(String.format("获取地区的商圈信息：%s", url));
         Map<String, String> bizMap = new HashMap<>();

src/main/webapp/app/dydb/data/dataLianjia/dataLianjia.js

@@ -28,18 +28,18 @@
             get: {method: 'GET', params: {method: 'get', id: '@id'}, isArray: false},
 
             // 获得行政区域数据
-            getCityMap: {method: 'GET', params: {method: 'citymap'}, isArray: false},
+            getCityMap: {method: 'GET', params: {method: 'citymap',parentId:'@parentId'}, isArray: false},
 
             // 获得商圈数据
-            getBizMap: {method: 'GET', params: {method: 'citymap-biz', code: '@code'}, isArray: false},
+            getBizMap: {method: 'GET', params: {method: 'citymap-biz', code: '@code' , cityCode: '@cityCode'}, isArray: false},
 
             // 拉取数据
-            fetch: {method: 'POST', params: {method: 'fetch'}, isArray: false},
+            fetch: {method: 'POST', params: {method: 'fetch'}, isArray: false,timeout: 20000},
 
             // 分页查询
             pageQuery: {
                 method: 'POST',
-                params: {method: 'pageQuery', limit: '@limit', start: '@start', orderBy: '@orderBy'},
+                params: {method: 'pageQuery', limit: '@limit', cityCode: '@cityCode',start: '@start', orderBy: '@orderBy'},
                 isArray: false
             },
 

src/main/webapp/app/dydb/data/dataLianjia/dataLianjia_list.js

@@ -13,26 +13,52 @@
             orderBy: '-createdDatetime,-dealDate,areaName,houses'
         };
 
-        $scope.condition = angular.extend({}, defaults);
+        $scope.condition = angular.extend({cityCode:'cd'}, defaults);
 
         $scope.fetch = {page: 1, endPage: 100};
+        // 加载城市信息
+        $scope.citys = [];
+        DataLianjiaService.getCityMap({parentId: '0'},function (o) {
+            angular.forEach(o.data, function (data) {
+                $scope.citys.push({name: data.name, value: data.id, code: data.code});
+            });
+        });
+
+        $scope.cityChange = function(){
+            $scope.areas = [];
+            $scope.condition.areaName = undefined;
+            if ($scope.condition.cityCode) {
+                angular.forEach($scope.citys, function (o) {
+                    if (o.code.indexOf($scope.condition.cityCode) === 0) {
+                        // 获取code
+                        DataLianjiaService.getCityMap({parentId: o.value}, function (bizData) {
+                            angular.forEach(bizData.data, function (data) {
+                                $scope.areas.push({name: data.name, value: data.id, code: data.code});
+                            });
+                        });
+                    }
+                });
+            }
+            $scope.query();
+        }
 
-        // 加载行政区域信息
-        $scope.areas = [{name: '全部'}];
-        DataLianjiaService.getCityMap(function (o) {
-            angular.forEach(o.data, function (value, code) {
-                $scope.areas.push({name: value, value: value, code: code});
+        //加载行政区域信息
+        $scope.areas = [];
+        DataLianjiaService.getCityMap({parentId: '1780490511923245056'},function (o) {
+            angular.forEach(o.data, function (data) {
+                $scope.areas.push({name: data.name, value: data.name, code: data.code});
             });
         });
-        $scope.bizs = [{name: '全部'}];
+        $scope.bizs = [];
         $scope.areaChange = function () {
-            $scope.bizs = [{name: '全部'}];
+            $scope.bizs = [];
             $scope.condition.bizName = undefined;
-            if ($scope.condition.areaName) {
+            var cityCode = $scope.condition.cityCode;
+            if ($scope.condition.areaName && $scope.condition.areaName != '全部') {
                 angular.forEach($scope.areas, function (o) {
                     if (o.name.indexOf($scope.condition.areaName) === 0) {
                         // 获取code
-                        DataLianjiaService.getBizMap({code: o.code}, function (bizData) {
+                        DataLianjiaService.getBizMap({code: o.code,cityCode:cityCode}, function (bizData) {
                             angular.forEach(bizData.data, function (bizName, bizCode) {
                                 $scope.bizs.push({name: bizName, value: bizName, code: bizCode});
                             });
@@ -102,6 +128,7 @@
         };
 
         $scope.fetchData = function () {
+
             // 判断是否有行政区域
             var areaCode = '';
             if ($scope.condition.areaName) {
@@ -126,6 +153,7 @@
             var params = angular.extend({}, $scope.fetch);
             params.page = params.page || 1;
             params.endPage = params.endPage || 100;
+            params.cityCode = $scope.condition.cityCode;
             if (bizCode) {
                 params.areaCode = areaCode;
                 params.bizCode = bizCode;

src/main/webapp/app/dydb/data/dataLianjia/dataLianjia_list.jsp

@@ -34,10 +34,18 @@
                         </div>
                         <div class="item w200">
                             <div class="form-label w80">
+                                <label>城市</label>
+                            </div>
+                            <select class="w120" ng-model="condition.cityCode"
+                                    ng-options="foo.code as foo.name for foo in citys"
+                                    ng-change="cityChange();"> </select>
+                        </div>
+                        <div class="item w200">
+                            <div class="form-label w80">
                                 <label>行政区</label>
                             </div>
                             <select class="w120" ng-model="condition.areaName"
-                                    ng-options="foo.value as foo.name for foo in areas"
+                                    ng-options="foo.name as foo.name for foo in areas"
                                     ng-change="areaChange();"> </select>
                         </div>
                         <div class="item w200">
@@ -164,20 +172,22 @@
                                          anyone-selected="anyone"></div>
                                 </td>
                                 <td style="width: 20px;">序号</td>
-                                <td>区</td>
+                                <td>城市</td>
+                                <td>行政区</td>
+                                <td>商圈</td>
                                 <td>楼盘/小区</td>
-                                <td>坐落</td>
+<%--                                <td>坐落</td>--%>
                                 <td>成交日期</td>
                                 <td>成交总价(万)</td>
                                 <td>成交单价(元)</td>
                                 <td>建筑面积</td>
-                                <td>套内面积</td>
-                                <td>楼层</td>
-                                <td>建筑类型</td>
-                                <td>房屋朝向</td>
-                                <td>建筑年代</td>
-                                <td>装修情况</td>
-                                <td>电梯</td>
+<%--                                <td>套内面积</td>--%>
+<%--                                <td>楼层</td>--%>
+<%--                                <td>建筑类型</td>--%>
+<%--                                <td>房屋朝向</td>--%>
+<%--                                <td>建筑年代</td>--%>
+<%--                                <td>装修情况</td>--%>
+<%--                                <td>电梯</td>--%>
                                 <td>抓取时间</td>
                                 <td>操作</td>
                             </tr>
@@ -189,22 +199,24 @@
                             <tr bindonce ng-repeat="foo in beans.data" ng-cloak>
                                 <td><input type="checkbox" ng-model="foo.isSelected"/></td>
                                 <td bo-text="pager.start+$index+1"></td>
+                                <td bo-text="foo.cityName"></td>
                                 <td bo-text="foo.areaName"></td>
+                                <td bo-text="foo.bizName"></td>
                                 <td>
                                     <a ng-click="view(foo.url)" bo-text="foo.houses" class="cp" title="点击查看详情"></a>
                                 </td>
-                                <td bo-text="foo.street||foo.address"></td>
+<%--                                <td bo-text="foo.street||foo.address"></td>--%>
                                 <td bo-text="foo.dealDate|eccrmDate"></td>
                                 <td bo-text="foo.dealMoney|number:2"></td>
                                 <td bo-text="foo.dealPrice|number:2"></td>
                                 <td bo-text="foo.buildArea|number:2"></td>
-                                <td bo-text="foo.roomArea|number:2"></td>
-                                <td bo-text="foo.floorInfo"></td>
-                                <td bo-text="foo.buildType"></td>
-                                <td bo-text="foo.orientation"></td>
-                                <td bo-text="foo.buildYear"></td>
-                                <td bo-text="foo.decorate"></td>
-                                <td bo-text="foo.lift"></td>
+<%--                                <td bo-text="foo.roomArea|number:2"></td>--%>
+<%--                                <td bo-text="foo.floorInfo"></td>--%>
+<%--                                <td bo-text="foo.buildType"></td>--%>
+<%--                                <td bo-text="foo.orientation"></td>--%>
+<%--                                <td bo-text="foo.buildYear"></td>--%>
+<%--                                <td bo-text="foo.decorate"></td>--%>
+<%--                                <td bo-text="foo.lift"></td>--%>
                                 <td bo-text="foo.createdDatetime|eccrmDate"></td>
                                 <td class="text-left">
                                     <a class="btn-op blue" ng-click="modify(foo.id);">编辑</a>