rewrite method of generating IPs

.github/workflows/build.yml

@@ -0,0 +1,43 @@
+name: Build
+
+on:
+  push:
+    branches: [ '*' ]
+  pull_request:
+    branches: [ '*' ]
+
+  workflow_dispatch:
+  schedule:
+    - cron: '30 4 * * *'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      BGPTOOLS_VERSION: 0.0.3
+      GO111MODULE: on
+      BVIEW_PATH: /mnt/tmp/
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
+        with:
+          go-version: 1.19
+      - run: sudo apt-get install -y bgpdump build-essential git
+      - uses: actions/cache@v3
+        with:
+          key: ${{ runner.os }}-build-dependencies
+          path: |
+            ~/go/bin
+      - run: ./dependency.sh
+      - run: sudo mkdir -p /mnt/tmp && sudo chmod 777 /mnt/tmp
+      - run: ./generate.sh
+      - run: ./stat.sh
+      - run: ./guard.sh
+      - name: Checkout ip-lists branch
+        uses: actions/checkout@v3
+        if: github.event_name == 'schedule' && github.ref == 'refs/heads/master'
+        with:
+          ref: ip-lists
+          path: ip-lists
+      - run: ./upload.sh
+        if: github.event_name == 'schedule' && github.ref == 'refs/heads/master'

common.sh

@@ -6,38 +6,12 @@ log_info(){
 	>&2 echo "INFO>" $@
 }
 
-get_asn(){
-	local CONF_FILE=$1
-	unset PATTERN
-	unset COUNTRY
-	unset EXCLUDE
-	source $CONF_FILE
-	EXCLUDE=${EXCLUDE:-"^$"}
-	grep -P "${COUNTRY}\$" asnames.txt |
-	grep -Pi "$PATTERN" |
-	grep -vPi "$EXCLUDE" |
-	awk '{gsub(/AS/, ""); print $1 }'
-}
-
-prepare_data_v4(){
-	curl -sSLo rib.bz2 http://archive.routeviews.org/dnszones/rib.bz2
-	log_info "runing bgpdump v4 ..."
-	bgpdump -m -O rib.txt rib.bz2
-	log_info "finish bgpdump v4"
-}
-prepare_data_v6(){
-	IP6UPSTREAM="http://archive.routeviews.org/route-views6/bgpdata"
-	MONTH6=$(date -u +%Y.%m)
-	LATEST6=$(lftp -e 'cls -1;exit' $IP6UPSTREAM/$MONTH6/RIBS/  2>/dev/null | sort | tail -n 1)
-	curl -sSLo rib6.bz2 "$IP6UPSTREAM/$MONTH6/RIBS/$LATEST6"
-	log_info "runing bgpdump v6 ..."
-	bgpdump -m -O rib6.txt rib6.bz2
-	log_info "finish bgpdump v6"
-}
 prepare_data(){
-	curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt &
-	prepare_data_v4 &
-	prepare_data_v6 &
+	curl -sSLo latest-bview.gz https://data.ris.ripe.net/rrc00/latest-bview.gz
+	log_info "runing bgpdump ..."
+	bgpdump -m -O ${BVIEW_PATH}latest-bview.txt latest-bview.gz
+	grep -v '::/' < ${BVIEW_PATH}latest-bview.txt > rib.txt & # prepare_data_v4
+	grep '::/' < ${BVIEW_PATH}latest-bview.txt > rib6.txt & # prepare_data_v6
 	wait_exit
 }
 

dependency.sh

@@ -3,13 +3,13 @@
 set -e
 
 cidr-merger --version || {
-    curl -sL -o ~/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
-    chmod +x ~/bin/gimme
-    eval "$(gimme stable)"
-    go get github.com/zhanhb/cidr-merger
+    go install github.com/zhanhb/[email protected]
+}
+asroute 0 < /dev/null || {
+    git clone https://github.com/yangzhaofeng/aspathanalysis.git
+    cd aspathanalysis
+    make && sudo make install
 }
-bgptools --version | grep -F $BGPTOOLS_VERSION || \
-    cargo install --vers $BGPTOOLS_VERSION bgptools
 
 cidr-merger --version
-bgptools --version
+asroute 0 < /dev/null

generate.sh

@@ -8,9 +8,18 @@ for file in operator/*.conf; do
 	operator=${file%.*}
 	operator=${operator##*/}
 	log_info "generating IP list of $operator ..."
-	get_asn $file
-	get_asn $file | xargs bgptools -b rib.txt  | cidr-merger -s > result/${operator}.txt &
-	get_asn $file | xargs bgptools -b rib6.txt | cidr-merger -s > result/${operator}6.txt &
+	source $file
+	if [ -z "$AS6" ]; then
+		AS6=$AS
+	fi
+	asroute $AS $EXCLUDE < rib.txt | cidr-merger -s > result/${operator}.txt &
+	asroute $AS6 $EXCLUDE < rib6.txt | cidr-merger -s > result/${operator}6.txt &
+	unset AS AS6 EXCLUDE
 done
-
 wait_exit
+
+{
+	cd result/
+	ls ../operator/ | sed 's/\.conf/.txt/g' | xargs cat | cidr-merger -s > china.txt
+	ls ../operator/ | sed 's/\.conf/6.txt/g' | xargs cat | cidr-merger -s > china6.txt
+}

guard.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+[[ $(wc -l result/china.txt) < 3000 ]] && exit 1
+
+[[ $(wc -l result/china6.txt) < 1000 ]] && exit 2
+
+exit 0

operator/cernet.conf

@@ -1,2 +1,3 @@
-PATTERN='(cngi|cernet)'
-COUNTRY='CN'
+AS=4538
+AS6=23911
+EXCLUDE="4134 4837 7497 9808 9394 9929"

operator/chinanet.conf

@@ -1,2 +1,2 @@
-PATTERN='(china ?telecom|chinanet|AS4847 )'
-COUNTRY='CN'
+AS=4134
+EXCLUDE="4538 4837 7497 9808 9394 9929 23910"

operator/cmcc.conf

@@ -1,2 +1,2 @@
-PATTERN='china mobile|cmnet'
-COUNTRY='CN'
+AS=9808
+EXCLUDE="4134 4538 4837 7497 9394 9929 23910"

operator/cstnet.conf

@@ -1,2 +1,2 @@
-PATTERN='cstnet'
-COUNTRY='CN'
+AS=7497
+EXCLUDE="4134 4538 4837 9808 9394 9929 23910"

operator/netcom.conf

@@ -0,0 +1,2 @@
+AS=9929
+EXCLUDE="4134 4538 4837 7497 9808 9394 23910"

operator/tietong.conf

@@ -1,2 +1,2 @@
-PATTERN='(tietong|CHINA RAILWAY)'
-COUNTRY='CN'
+AS=9394
+EXCLUDE="4134 4538 4837 7497 9808 9929 23910"

operator/unicom.conf

@@ -1,2 +1,2 @@
-PATTERN='(unicom|cnc)'
-COUNTRY='CN'
+AS=4837
+EXCLUDE="4134 4538 7497 9808 9394 9929 23910"

upload.sh

@@ -1,12 +1,11 @@
 #!/usr/bin/env bash
 
-git clone -b ip-lists https://${GH_REF} ip-lists
+rm ip-lists/*.txt
 mv result/* ip-lists
 cd ip-lists
-git config user.name $GIT_USER_NAME
-git config user.email $GIT_USER_EMAIL
+git config user.name Github Actions
+git config user.email [email protected]
 git add .
 git commit -m "update $(date +%Y-%m-%d)"
-git push -q "https://${GH_TOKEN}@${GH_REF}" ip-lists:ip-lists
-git push -q "https://${GH_TOKEN}@${GH_REF}" ip-lists:gh-pages
+git push -q