#!/bin/sh
curl -o users.json "https://codeforces.net/api/user.ratedList?activeOnly=false"

#!/bin/sh

url='https://atcoder.jp/ranking/all?page='
filename='ranking'
fileext='.html'
downloader='curl -o'

echo $$$downloader $$${filename}1$fileext "${url}1"
$$$downloader $$${filename}1$fileext "${url}1"

num=`grep -P -o 'page=\d+' ranking1.html | grep -P -o '\d+' | sort -n | tail -n 1`

for ((i=2;i<=$num;i++)); do
    echo $downloader $filename$i$fileext "$url$i"
    sleep 10s
    $downloader $filename$i$fileext "$url$i"
done

package main

import (
	"bufio"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type User struct {
	OverallRank  int    `json:"overallrank"`
	Rank         int    `json:"Rank"`
	Country      string `json:"country"`
	Handle       string `json:"handle"`
	Organization string `json:"organization"`
	Birth        int    `json:"birth"`
	Rating       int    `json:"rating"`
	Highest      int    `json:"highest"`
	Matches      int    `json:"matches"`
	Wins         int    `json:"wins"`
}

func (u *User) String() string {
	return fmt.Sprintf("%#v", *u)
}

func main() {

	users := []*User{}

	files, err := filepath.Glob("ranking*.html")
	if err != nil {
		panic(err)
	}

	for num := range files {

		file, err := os.Open(files[num])
		if err != nil {
			panic(err)
		}
		defer file.Close()

		red, wri := io.Pipe()

		go func() {
			defer wri.Close()
			bufRed := bufio.NewReader(file)
			bufWri := bufio.NewWriter(wri)
			for {
				ch, _, err := bufRed.ReadRune()
				if err != nil {
					bufWri.Flush()
					wri.CloseWithError(err)
					return
				}
				if ch == 0x10 {
					ch = '+'
				}
				if _, err = bufWri.WriteRune(ch); err != nil {
					wri.CloseWithError(err)
					return
				}
			}
		}()

		dec := xml.NewDecoder(bufio.NewReader(red))
		dec.Strict = false
		dec.AutoClose = xml.HTMLAutoClose
		dec.Entity = xml.HTMLEntity

		for {
			if se, err := skipToSE(dec, "table"); err != nil {
				panic(err)
			} else if len(se.(xml.StartElement).Attr) == 0 {
				continue
			}
			if _, err := skipToSE(dec, "tbody"); err != nil {
				panic(err)
			}
			break
		}

		for {
			if token, err := skipNoElm(dec); err != nil {
				panic(err)
			} else if ee, ok := token.(xml.EndElement); ok && ee.Name.Local == "tbody" {
				break
			} else if se, ok := token.(xml.StartElement); !ok || se.Name.Local != "tr" {
				continue
			}
			user := &User{}
			for i := 0; i < 7; i++ {
				if _, err := skipToSE(dec, "td"); err != nil {
					println(fmt.Sprintf("num: %d", num))
					panic(err)
				}
				switch i {
				case 0:
					if ovR, err := getCharData(dec, "span"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						ovR := strings.Trim(ovR, "()")
						if overallRank, err := strconv.Atoi(ovR); err == nil {
							user.OverallRank = overallRank
						}
					}
					if r, err := getCharData(dec, "td"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						if rank, err := strconv.Atoi(r); err == nil {
							user.Rank = rank
						}
					}
				case 1:
					if se, err := skipToSE(dec, "a"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						se := se.(xml.StartElement)
						for _, attr := range se.Attr {
							if attr.Name.Local != "href" {
								continue
							}
							if ss := strings.Split(attr.Value, "="); len(ss) == 2 {
								user.Country = ss[1]
							}
							break
						}
					}
					if handle, err := getCharData(dec, "span"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						user.Handle = handle
					}
					if org, err := getCharData(dec, "td"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						user.Organization = org
					}
				case 2:
					if year, err := getCharData(dec, "td"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						if year, err := strconv.Atoi(year); err == nil {
							user.Birth = year
						}
					}
				case 3:
					if rate, err := getCharData(dec, "td"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						if rate, err := strconv.Atoi(rate); err == nil {
							user.Rating = rate
						}
					}
				case 4:
					if maxrate, err := getCharData(dec, "td"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						if maxrate, err := strconv.Atoi(maxrate); err == nil {
							user.Highest = maxrate
						}
					}
				case 5:
					if comp, err := getCharData(dec, "td"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						if comp, err := strconv.Atoi(comp); err == nil {
							user.Matches = comp
						}
					}
				case 6:
					if wins, err := getCharData(dec, "td"); err != nil {
						println(fmt.Sprintf("num: %d", num))
						panic(err)
					} else {
						if wins, err := strconv.Atoi(wins); err == nil {
							user.Wins = wins
						}
					}
				}
			}
			users = append(users, user)
		}
	}

	dst, err := os.Create("users.json")
	if err != nil {
		panic(err)
	}
	defer dst.Close()
	out := bufio.NewWriter(dst)
	defer out.Flush()
	json.NewEncoder(out).Encode(users)
}

func skipToSE(dec *xml.Decoder, tag string) (xml.Token, error) {
	for {
		token, err := dec.Token()
		if err != nil {
			return token, err
		}
		if se, ok := token.(xml.StartElement); ok && se.Name.Local == tag {
			return token, nil
		}
	}
}

func getCharData(dec *xml.Decoder, endTag string) (string, error) {
	ret := ""
	for {
		token, err := dec.Token()
		if err != nil {
			println(fmt.Sprintf("%#v", token))
			println(fmt.Sprintf("%#v", err))
			tok2, err2 := dec.RawToken()
			println(fmt.Sprintf("%#v", tok2))
			println(fmt.Sprintf("%#v", err2))
			return ret, err
		}
		switch token := token.(type) {
		case xml.EndElement:
			if token.Name.Local == endTag {
				return strings.TrimSpace(ret), nil
			}
		case xml.CharData:
			ret += string(token)
		}
	}
}

func skipNoElm(dec *xml.Decoder) (xml.Token, error) {
	for {
		token, err := dec.Token()
		if err != nil {
			return nil, err
		}
		switch token.(type) {
		case xml.StartElement, xml.EndElement:
			return token, nil
		}
	}
}

package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

type AtCoderUser struct {
	OverallRank  int    `json:"overallrank"`
	Rank         int    `json:"Rank"`
	Country      string `json:"country"`
	Handle       string `json:"handle"`
	Organization string `json:"organization"`
	Birth        int    `json:"birth"`
	Rating       int    `json:"rating"`
	Highest      int    `json:"highest"`
	Matches      int    `json:"matches"`
	Wins         int    `json:"wins"`
}

type CodeforcesUser struct {
	Handle                  string
	Email                   string
	VkId                    string
	OpenId                  string
	FirstName               string
	LastName                string
	Country                 string
	City                    string
	Organization            string
	Contribution            int
	Rank                    string
	Rating                  int
	MaxRank                 string
	MaxRating               int
	LastOnlineTimeSeconds   int
	RegistrationTimeSeconds int
	FriendOfCount           int
	Avatar                  string
	TitlePhoto              string
}

type CodeforcesUserRatedList struct {
	Status string
	Result []*CodeforcesUser
}

type CodeforcesRatingChange struct {
	ContestId               int
	ContestName             string
	Handle                  string
	Rank                    int
	RatingUpdateTimeSeconds int
	OldRating               int
	NewRating               int
}

type CodeforcesUserRating struct {
	Status string
	Result []*CodeforcesRatingChange
}

func LoadJson(filepath string, data interface{}) error {

	file, err := os.Open(filepath)
	if err != nil {
		return err
	}
	defer file.Close()

	dec := json.NewDecoder(bufio.NewReader(file))

	if err := dec.Decode(data); err != nil {
		return err
	}

	return nil
}

func GetCfRating(handle string) (*CodeforcesUserRating, error) {
	url := "https://codeforces.net/api/user.rating?handle=" + url.QueryEscape(handle)
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	rating := &CodeforcesUserRating{}
	err = json.Unmarshal(data, rating)
	return rating, err
}

func main() {

	var atcoderUsers []*AtCoderUser

	if err := LoadJson(filepath.Join("atcoder", "users.json"), &atcoderUsers); err != nil {
		panic(err)
	}

	sort.Slice(atcoderUsers, func(i, j int) bool {
		return strings.Compare(atcoderUsers[i].Handle, atcoderUsers[j].Handle) < 0
	})

	var ratedList CodeforcesUserRatedList

	if err := LoadJson(filepath.Join("codeforces", "users.json"), &ratedList); err != nil {
		panic(err)
	}

	if ratedList.Status != "OK" {
		println("failed downloading ratedList")
		return
	}

	dst, err := os.Create("coders.csv")
	if err != nil {
		panic(err)
	}
	defer dst.Close()

	wri := csv.NewWriter(dst)

	if err := wri.Write([]string{
		"handle",
		"atcoder_rating",
		"atcoder_maxrating",
		"atcoder_country",
		"codeforces_rating",
		"codeforces_maxrating",
		"codeforces_country",
		"codeforces_lastonline",
		"atcoder_matches",
		"codeforces_matches",
	}); err != nil {
		panic(err)
	}

	waiting := time.After(time.Second)
	count := 0

	for _, cfUser := range ratedList.Result {
		i := sort.Search(len(atcoderUsers), func(i int) bool {
			return strings.Compare(atcoderUsers[i].Handle, cfUser.Handle) >= 0
		})
		if 0 <= i && i < len(atcoderUsers) && atcoderUsers[i].Handle == cfUser.Handle {
			count++
			if count%60 == 0 {
				println("count " + strconv.Itoa(count))
			}
			<-waiting
			rating, err := GetCfRating(cfUser.Handle)
			if err != nil {
				panic(err)
			}
			if rating.Status != "OK" {
				println("failed downloading rating")
				return
			}
			waiting = time.After(time.Second)
			acUser := atcoderUsers[i]
			wri.Write([]string{
				cfUser.Handle,
				strconv.Itoa(acUser.Rating),
				strconv.Itoa(acUser.Highest),
				acUser.Country,
				strconv.Itoa(int(cfUser.Rating)),
				strconv.Itoa(int(cfUser.MaxRating)),
				cfUser.Country,
				time.Unix(int64(cfUser.LastOnlineTimeSeconds), 0).UTC().String(),
				strconv.Itoa(acUser.Matches),
				strconv.Itoa(len(rating.Result)),
			})
		}
	}

	wri.Flush()
	if err := wri.Error(); err != nil {
		panic(err)
	}
}

package main

import (
	"bufio"
	"encoding/csv"
	"flag"
	"fmt"
	"image"
	"image/color"
	"image/png"
	"os"
	"strconv"
)

var acColors = []color.RGBA{
	color.RGBA{0x80, 0x80, 0x80, 0xff},
	color.RGBA{0x80, 0x40, 0x00, 0xff},
	color.RGBA{0x00, 0x80, 0x00, 0xff},
	color.RGBA{0x00, 0xC0, 0xC0, 0xff},
	color.RGBA{0x00, 0x00, 0xFF, 0xff},
	color.RGBA{0xC0, 0xC0, 0x00, 0xff},
	color.RGBA{0xFF, 0x80, 0x00, 0xff},
	color.RGBA{0xFF, 0x00, 0x00, 0xff},
	color.RGBA{0xC0, 0xC0, 0xC0, 0xff},
	color.RGBA{0xFF, 0xD7, 0x00, 0xff},
}

var cfColors = []color.RGBA{
	color.RGBA{0x80, 0x80, 0x80, 0xff},
	color.RGBA{0x00, 0x80, 0x00, 0xff},
	color.RGBA{0x03, 0xA8, 0x9E, 0xff},
	color.RGBA{0x00, 0x00, 0xFF, 0xff},
	color.RGBA{0xAA, 0x00, 0xAA, 0xff},
	color.RGBA{0xFF, 0x8C, 0x00, 0xff},
	color.RGBA{0xFF, 0x00, 0x00, 0xff},
}

var cfRatingBorders = []int{
	1200,
	1400,
	1600,
	1900,
	2100,
	2400,
	3000,
}

func Max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func Min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func main() {

	var minMatches int
	flag.IntVar(&minMatches, "matches", 10, "min matches")
	flag.Parse()

	file, err := os.Open("coders.csv")
	if err != nil {
		panic(err)
	}
	defer file.Close()

	rows, err := csv.NewReader(bufio.NewReader(file)).ReadAll()
	if err != nil {
		panic(err)
	}

	dt := []int{-1, 0, 1, 0, -1}

	for p := 0; p < 2; p++ {
		cnt := make(map[int]int)

		minAcRating := 9999
		maxAcRating := -9999
		minCfRating := 9999
		maxCfRating := -9999

		for _, row := range rows[1:] {
			acRating, err := strconv.Atoi(row[1+p])
			if err != nil {
				panic(err)
			}
			cfRating, err := strconv.Atoi(row[4+p])
			if err != nil {
				panic(err)
			}

			acMatches, err := strconv.Atoi(row[8])
			if err != nil {
				panic(err)
			}
			cfMatches, err := strconv.Atoi(row[9])
			if err != nil {
				panic(err)
			}
			if acMatches < minMatches || cfMatches < minMatches {
				continue
			}

			minAcRating = Min(minAcRating, acRating)
			maxAcRating = Max(maxAcRating, acRating)
			minCfRating = Min(minCfRating, cfRating)
			maxCfRating = Max(maxCfRating, cfRating)

			acRating += 1000
			acRating /= 5
			cfRating += 1000
			cfRating /= 5
			key := acRating*10000 + cfRating
			if c, ok := cnt[key]; ok {
				cnt[key] = c + 1
			} else {
				cnt[key] = 1
			}
		}
		fmt.Println("minAcRating", minAcRating)
		fmt.Println("maxAcRating", maxAcRating)
		fmt.Println("minCfRating", minCfRating)
		fmt.Println("maxCfRating", maxCfRating)
		fmt.Println("uniq", len(cnt))

		max := 0
		for _, c := range cnt {
			if c > max {
				max = c
			}
		}
		fmt.Println("maxCount", max)

		img := image.NewRGBA(image.Rect(0, 0, 1000, 1000))

		for k, c := range cnt {
			acRating := k / 10000
			cfRating := k % 10000
			r := uint8(255 * c / max)
			img.Set(cfRating, 999-acRating, color.RGBA{r, 0, 0, 0xff})
			r = r * 95 / 100
			for i := 0; i < 4; i++ {
				ac := acRating + dt[i]
				cf := cfRating + dt[i+1]
				if ac < 0 || ac >= 1000 || cf < 0 || cf >= 1000 {
					continue
				}
				if _, _, _, a := img.At(cf, 999-ac).RGBA(); a == 0 {
					img.Set(cf, 999-ac, color.RGBA{r, 0, 0, 0xff})
				}
			}
		}

		gray := color.RGBA{60, 60, 60, 0xff}
		for i := 0; i < 1000; i++ {
			if _, _, _, a := img.At(200, i).RGBA(); a == 0 {
				img.Set(200, i, gray)
			}
			if _, _, _, a := img.At(i, 999-200).RGBA(); a == 0 {
				img.Set(i, 999-200, gray)
			}
		}

		lightgray1 := color.RGBA{190, 190, 190, 0xff}
		lightgray2 := color.RGBA{235, 235, 235, 0xff}
		for i := 0; i < 800; i++ {
			for j := 0; j < 800; j += 20 {
				if _, _, _, a := img.At(200+i, 999-200-j).RGBA(); a == 0 {
					if j%80 == 0 {
						img.Set(200+i, 999-200-j, lightgray1)
					} else {
						img.Set(200+i, 999-200-j, lightgray2)
					}
				}
				if _, _, _, a := img.At(200+j, 999-200-i).RGBA(); a == 0 {
					col := lightgray2
					for _, r := range cfRatingBorders {
						if r/5 == j {
							col = lightgray1
							break
						}
					}
					img.Set(200+j, 999-200-i, col)
				}
			}
		}

		for i, col := range acColors {
			for x := 190; x < 200; x++ {
				for dy := 0; dy < 80; dy++ {
					y := i*80 + dy
					if _, _, _, a := img.At(x, 999-200-y).RGBA(); a == 0 {
						img.Set(x, 999-200-y, col)
					}
				}
			}
		}

		for y := 190; y < 200; y++ {
			for x := 0; x < 800; x++ {
				if _, _, _, a := img.At(200+x, 999-y).RGBA(); a != 0 {
					continue
				}
				col := color.RGBA{0, 0, 0, 0xff}
				for i, r := range cfRatingBorders {
					if x < r/5 {
						col = cfColors[i]
						break
					}
				}
				img.Set(200+x, 999-y, col)
			}
		}

		white := color.RGBA{0xFF, 0xFF, 0xFF, 0xff}
		for y := 0; y < 1000; y++ {
			for x := 0; x < 1000; x++ {
				if _, _, _, a := img.At(x, y).RGBA(); a == 0 {
					img.Set(x, y, white)
				}
			}
		}

		graph, err := os.Create(fmt.Sprintf("graph%d_%d.png", minMatches, p))
		if err != nil {
			panic(err)
		}
		defer graph.Close()
		err = png.Encode(graph, img)
		if err != nil {
			panic(err)
		}
	}
}