abfmigrator/main.go
2024-11-25 23:25:55 +03:00

798 lines
24 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package main
import (
"bytes"
"encoding/base64"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
"github.com/joho/godotenv"
)
// AbfJson структура для взаимодействия с API ABF.io
type AbfJson struct {
AbfURL string
FileStoreURL string
Login string
Password string
Base64AuthString string
Log *log.Logger
CacheData map[string][]byte
CacheEtags map[string]string
}
// Ошибки API ABF.io
var errorsMap = map[string]error{
"Invalid email or password.": fmt.Errorf("Authentication error"),
"403 Forbidden | Rate Limit Exceeded": fmt.Errorf("Rate limit exceeded"),
"Page not found": fmt.Errorf("Page not found"),
"Project has not been forked. Name has already been taken": fmt.Errorf("Name already taken"),
"Error 404. Resource not found!": fmt.Errorf("Resource not found"),
"Something went wrong. We've been notified about this issue and we'll take a look at it shortly.": fmt.Errorf("Internal server error"),
"We update the site, it will take some time. We are really trying to do it fast. We apologize for any inconvenience..": fmt.Errorf("Server under maintenance"),
"Requires authentication": fmt.Errorf("Authentication required"),
"Forbidden. Sorry, you don't have enough rights for this action!": fmt.Errorf("Forbidden"),
"Access violation to this page!": fmt.Errorf("Access violation"),
"Bad Request": fmt.Errorf("Bad request"),
}
// GoodMessages сообщения, которые не считаются ошибками
var goodMessages = []string{
"Errors during build publishing!",
"Build is queued for publishing",
}
// FatalErrors фатальные ошибки, которые приводят к выходу из программы
var fatalErrors = []error{
errorsMap["Invalid email or password."],
errorsMap["403 Forbidden | Rate Limit Exceeded"],
errorsMap["Something went wrong. We've been notified about this issue and we'll take a look at it shortly."],
}
// Project структура для хранения информации о проекте
type Project struct {
ID int `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
}
// ProjectDetails структура для хранения деталей проекта
type ProjectDetails struct {
ID int `json:"id"`
Name string `json:"name"`
URL string `json:"git_url"`
WebURL string `json:"web_url"`
SSHURLToRepo string `json:"ssh_url_to_repo"`
HTTPURLToRepo string `json:"http_url_to_repo"`
}
// CacheEntry структура для хранения данных кеша
type CacheEntry struct {
Data map[string][]byte `json:"data"`
CacheEtags map[string]string `json:"cache_etags"`
Timestamp time.Time `json:"timestamp"`
}
// NewAbfJson создает новый экземпляр AbfJson
func NewAbfJson(abfURL, fileStoreURL, login, password string, log *log.Logger) (*AbfJson, error) {
if !strings.HasPrefix(fileStoreURL, "http://") && !strings.HasPrefix(fileStoreURL, "https://") {
log.Fatalf("File-store URL has to start with 'http(s)://'")
}
lpw := fmt.Sprintf("%s:%s", login, password)
encodedLpw := base64.StdEncoding.EncodeToString([]byte(lpw))
return &AbfJson{
AbfURL: strings.TrimSuffix(abfURL, "/"),
FileStoreURL: strings.TrimSuffix(fileStoreURL, "/"),
Login: login,
Password: password,
Base64AuthString: encodedLpw,
Log: log,
CacheData: make(map[string][]byte),
CacheEtags: make(map[string]string),
}, nil
}
// LoadCache загружает кеш из файла
func (a *AbfJson) LoadCache(cacheFilePath string) error {
data, err := ioutil.ReadFile(cacheFilePath)
if err != nil {
if os.IsNotExist(err) {
a.Log.Println("Cache file does not exist. Proceeding without cache.")
return nil
}
return fmt.Errorf("failed to read cache file: %v", err)
}
var cacheEntry CacheEntry
if err := json.Unmarshal(data, &cacheEntry); err != nil {
return fmt.Errorf("failed to unmarshal cache data: %v", err)
}
a.CacheData = cacheEntry.Data
a.CacheEtags = cacheEntry.CacheEtags
a.Log.Println("Cache loaded successfully.")
return nil
}
// SaveCache сохраняет кеш в файл
func (a *AbfJson) SaveCache(cacheFilePath string) error {
cacheEntry := CacheEntry{
Data: a.CacheData,
CacheEtags: a.CacheEtags,
Timestamp: time.Now(),
}
data, err := json.Marshal(cacheEntry)
if err != nil {
return fmt.Errorf("failed to marshal cache data: %v", err)
}
if err := ioutil.WriteFile(cacheFilePath, data, 0644); err != nil {
return fmt.Errorf("failed to write cache file: %v", err)
}
a.Log.Println("Cache saved successfully.")
return nil
}
// IsCacheValid проверяет, действителен ли кеш
func (a *AbfJson) IsCacheValid(cacheFilePath string, ttl time.Duration) (bool, error) {
data, err := ioutil.ReadFile(cacheFilePath)
if err != nil {
if os.IsNotExist(err) {
a.Log.Println("Cache file does not exist. Cache is invalid.")
return false, nil
}
return false, fmt.Errorf("failed to read cache file: %v", err)
}
var cacheEntry CacheEntry
if err := json.Unmarshal(data, &cacheEntry); err != nil {
return false, fmt.Errorf("failed to unmarshal cache data: %v", err)
}
if time.Since(cacheEntry.Timestamp) > ttl {
a.Log.Println("Cache has expired.")
return false, nil
}
a.Log.Println("Cache is valid.")
return true, nil
}
// ProcessResponse обрабатывает ответ API ABF.io
func (a *AbfJson) ProcessResponse(responseString []byte) (map[string]interface{}, error) {
var res map[string]interface{}
if err := json.Unmarshal(responseString, &res); err != nil {
a.Log.Fatalf("Internal server error: it has returned non-json data.")
}
message, ok := res["message"].(string)
if ok && !contains(goodMessages, message) {
return nil, errorsMap[message]
}
repository, repoOk := res["repository"].(map[string]interface{})
project, projOk := res["project"].(map[string]interface{})
if repoOk {
repoMessage, ok := repository["message"].(string)
if ok && (strings.Contains(repoMessage, "error") || strings.Contains(repoMessage, "has not been")) {
return nil, errorsMap[repoMessage]
}
}
if projOk {
projMessage, ok := project["message"].(string)
if ok && (strings.Contains(projMessage, "error") || strings.Contains(projMessage, "has not been")) {
return nil, errorsMap[projMessage]
}
}
if errStr, ok := res["error"].(string); ok && !contains(goodMessages, errStr) {
return nil, errorsMap[errStr]
}
return res, nil
}
// GetURLContents выполняет HTTP-запрос к API ABF.io
func (a *AbfJson) GetURLContents(path string, params map[string]string, method string, body interface{}) (interface{}, error) {
url := a.AbfURL + path
if method == "file_store" {
url = a.FileStoreURL + path
}
if len(params) > 0 {
query := ""
for key, value := range params {
if query == "" {
query += "?"
} else {
query += "&"
}
query += fmt.Sprintf("%s=%s", key, value)
}
url += query
}
a.Log.Printf("Fetching URL: %s", url)
var req *http.Request
var err error
var requestBody []byte
if body != nil {
requestBody, err = json.Marshal(body)
if err != nil {
a.Log.Fatalf("Failed to marshal request body: %v", err)
}
var prettyJSON bytes.Buffer
json.Indent(&prettyJSON, requestBody, "", " ")
a.Log.Printf("Request Body: %s", prettyJSON.String()) // Логирование отправляемого JSON
}
switch method {
case "POST":
req, err = http.NewRequest("POST", url, bytes.NewBuffer(requestBody))
case "PUT":
req, err = http.NewRequest("PUT", url, bytes.NewBuffer(requestBody))
case "DELETE":
req, err = http.NewRequest("DELETE", url, bytes.NewBuffer(requestBody))
default:
req, err = http.NewRequest("GET", url, nil)
}
if err != nil {
a.Log.Fatalf("Failed to create request: %v", err)
}
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", a.Base64AuthString))
req.Header.Add("Content-Type", "application/json")
etag, etagFound := a.CacheEtags[url]
if etagFound {
req.Header.Add("If-None-Match", etag)
}
client := &http.Client{
Timeout: 30 * time.Second,
}
resp, err := client.Do(req)
if err != nil {
a.Log.Fatalf("HTTP request failed: %v", err)
}
defer resp.Body.Close()
responseBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
a.Log.Fatalf("Failed to read response body: %v", err)
}
if resp.StatusCode == 304 && etagFound {
a.Log.Println("Getting cached result (cache was validated)")
return a.ProcessResponse(a.CacheData[etag])
}
if resp.StatusCode != 200 && resp.StatusCode != 201 {
a.Log.Printf("Return code: %d", resp.StatusCode)
return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(responseBody))
}
etagNew := resp.Header.Get("ETag")
if etagNew != "" {
a.Log.Printf("Caching the new value for %s. ETag is %s", url, etagNew)
a.CacheEtags[url] = etagNew
a.CacheData[etagNew] = responseBody
}
var res interface{}
if err := json.Unmarshal(responseBody, &res); err != nil {
a.Log.Fatalf("Failed to unmarshal response: %v", err)
}
return res, nil
}
// GetProjectsList получает список проектов с обходом всех страниц
func (a *AbfJson) GetProjectsList(platformID, repoID int, ownerName string) ([]Project, error) {
var allProjects []Project
page := 1
perPage := 100
for {
params := map[string]string{
"added": "true",
"format": "json",
"owner_name": ownerName,
"page": fmt.Sprintf("%d", page),
"per_page": fmt.Sprintf("%d", perPage),
"project_name": "",
}
path := fmt.Sprintf("/platforms/%d/repositories/%d/projects_list", platformID, repoID)
res, err := a.GetURLContents(path, params, "GET", nil)
if err != nil {
return nil, fmt.Errorf("Failed to get projects list: %v", err)
}
// Преобразуем res["projects"] в []byte
projectsArray, ok := res.(map[string]interface{})["projects"]
if !ok {
a.Log.Fatalf("Failed to find 'projects' in response")
}
projectsBytes, err := json.Marshal(projectsArray)
if err != nil {
a.Log.Fatalf("Failed to marshal projects list: %v", err)
}
var projects []Project
if err := json.Unmarshal(projectsBytes, &projects); err != nil {
a.Log.Fatalf("Failed to unmarshal projects list: %v", err)
}
allProjects = append(allProjects, projects...)
if len(projects) < perPage {
break
}
page++
}
return allProjects, nil
}
// GetProjectDetails получает детали проекта по ID
func (a *AbfJson) GetProjectDetails(projectID int) (*ProjectDetails, error) {
path := fmt.Sprintf("/api/v1/projects/%d.json", projectID)
res, err := a.GetURLContents(path, nil, "GET", nil)
if err != nil {
a.Log.Printf("Failed to get project details for project ID %d: %v", projectID, err)
return nil, err
}
projectData, ok := res.(map[string]interface{})["project"].(map[string]interface{})
if !ok {
a.Log.Printf("Missing 'project' field in response for project ID %d", projectID)
return nil, fmt.Errorf("Missing 'project' field in response")
}
id, idOk := projectData["id"]
if !idOk {
a.Log.Printf("Missing 'id' field in project details for project ID %d", projectID)
return nil, fmt.Errorf("Missing 'id' field in project details")
}
idFloat, idOk := id.(float64)
if !idOk {
a.Log.Printf("Invalid 'id' type in project details for project ID %d: expected float64, got %T", projectID, id)
return nil, fmt.Errorf("Invalid 'id' type in project details")
}
name, nameOk := projectData["name"].(string)
if !nameOk {
a.Log.Printf("Missing 'name' field in project details for project ID %d", projectID)
return nil, fmt.Errorf("Missing 'name' field in project details")
}
gitUrl, gitUrlOk := projectData["git_url"].(string)
webUrl, webUrlOk := projectData["web_url"].(string)
sshUrlToRepo, sshUrlToRepoOk := projectData["ssh_url_to_repo"].(string)
httpUrlToRepo, httpUrlToRepoOk := projectData["http_url_to_repo"].(string)
var url string
if gitUrlOk {
url = gitUrl
} else if httpUrlToRepoOk {
url = httpUrlToRepo
} else if webUrlOk {
url = webUrl
} else if sshUrlToRepoOk {
url = sshUrlToRepo
} else {
a.Log.Printf("Missing URL fields in project details for project ID %d", projectID)
return nil, fmt.Errorf("Missing URL fields in project details")
}
projectDetails := &ProjectDetails{
ID: int(idFloat),
Name: name,
URL: url,
WebURL: webUrl,
SSHURLToRepo: sshUrlToRepo,
HTTPURLToRepo: httpUrlToRepo,
}
return projectDetails, nil
}
// contains проверяет, содержит ли слайс строк строку
func contains(slice []string, str string) bool {
for _, item := range slice {
if item == str {
return true
}
}
return false
}
// GiteaClient структура для взаимодействия с API Gitea
type GiteaClient struct {
APIURL string
Token string
Owner string
Log *log.Logger
}
// Repository структура для хранения информации о репозитории в Gitea
type Repository struct {
ID int `json:"id"`
Name string `json:"name"`
}
// GetRepositories получает список репозиториев из Gitea
func (g *GiteaClient) GetRepositories() ([]Repository, error) {
path := "/user/repos"
res, err := g.GetURLContents(path, nil, "GET", nil)
if err != nil {
return nil, fmt.Errorf("Failed to get repositories: %v", err)
}
// Проверяем, что res является массивом
reposArray, ok := res.([]interface{})
if !ok {
g.Log.Fatalf("Expected array in response, got %T", res)
}
var repos []Repository
for _, repoInterface := range reposArray {
repoMap, ok := repoInterface.(map[string]interface{})
if !ok {
g.Log.Fatalf("Expected map in array element, got %T", repoInterface)
}
id, idOk := repoMap["id"].(float64)
if !idOk {
g.Log.Fatalf("Invalid 'id' type in repository: expected float64, got %T", repoMap["id"])
}
name, nameOk := repoMap["name"].(string)
if !nameOk {
g.Log.Fatalf("Missing 'name' field in repository")
}
repos = append(repos, Repository{
ID: int(id),
Name: name,
})
}
return repos, nil
}
// MirrorRepositoryPayload структура для запроса зеркалирования репозитория в Gitea
type MirrorRepositoryPayload struct {
AuthPassword string `json:"auth_password,omitempty"`
AuthToken string `json:"auth_token,omitempty"`
AuthUsername string `json:"auth_username,omitempty"`
CloneAddr string `json:"clone_addr"`
Description string `json:"description,omitempty"`
Issues bool `json:"issues"`
Labels bool `json:"labels"`
LFS bool `json:"lfs"`
LFSEndpoint string `json:"lfs_endpoint,omitempty"`
Milestones bool `json:"milestones"`
Mirror bool `json:"mirror"`
MirrorInterval string `json:"mirror_interval,omitempty"`
Private bool `json:"private"`
PullRequests bool `json:"pull_requests"`
Releases bool `json:"releases"`
RepoName string `json:"repo_name"`
RepoOwner string `json:"repo_owner"`
Service string `json:"service"`
UID int `json:"uid,omitempty"`
Wiki bool `json:"wiki"`
}
// MirrorRepository создает зеркало репозитория в Gitea
func (g *GiteaClient) MirrorRepository(repoName, cloneURL string) error {
payload := MirrorRepositoryPayload{
CloneAddr: cloneURL,
Description: "",
Issues: true,
Labels: true,
LFS: false,
Milestones: true,
Mirror: true,
MirrorInterval: "8h0m0s", // Интервал зеркалирования
Private: false,
PullRequests: true,
Releases: true,
RepoName: repoName,
RepoOwner: g.Owner,
Service: "git",
Wiki: true,
}
path := "/repos/migrate"
res, err := g.GetURLContents(path, nil, "POST", payload)
if err != nil {
return fmt.Errorf("Failed to mirror repository: %v", err)
}
if res == nil {
return fmt.Errorf("Empty response from Gitea")
}
g.Log.Printf("Response from Gitea: %+v", res)
return nil
}
// GetURLContents выполняет HTTP-запрос к API Gitea
func (g *GiteaClient) GetURLContents(path string, params map[string]string, method string, body interface{}) (interface{}, error) {
url := g.APIURL + path
if len(params) > 0 {
query := ""
for key, value := range params {
if query == "" {
query += "?"
} else {
query += "&"
}
query += fmt.Sprintf("%s=%s", key, value)
}
url += query
}
g.Log.Printf("Fetching URL: %s", url)
var req *http.Request
var err error
var requestBody []byte
if body != nil {
requestBody, err = json.Marshal(body)
if err != nil {
g.Log.Fatalf("Failed to marshal request body: %v", err)
}
var prettyJSON bytes.Buffer
json.Indent(&prettyJSON, requestBody, "", " ")
g.Log.Printf("Request Body: %s", prettyJSON.String()) // Логирование отправляемого JSON
}
switch method {
case "POST":
req, err = http.NewRequest("POST", url, bytes.NewBuffer(requestBody))
case "PUT":
req, err = http.NewRequest("PUT", url, bytes.NewBuffer(requestBody))
case "DELETE":
req, err = http.NewRequest("DELETE", url, bytes.NewBuffer(requestBody))
default:
req, err = http.NewRequest("GET", url, nil)
}
if err != nil {
g.Log.Fatalf("Failed to create request: %v", err)
}
req.Header.Add("Authorization", fmt.Sprintf("token %s", g.Token))
req.Header.Add("Content-Type", "application/json")
client := &http.Client{
Timeout: 30 * time.Second,
}
resp, err := client.Do(req)
if err != nil {
g.Log.Fatalf("HTTP request failed: %v", err)
}
defer resp.Body.Close()
responseBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
g.Log.Fatalf("Failed to read response body: %v", err)
}
if resp.StatusCode != 200 && resp.StatusCode != 201 {
g.Log.Printf("Return code: %d", resp.StatusCode)
return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(responseBody))
}
var res interface{}
if err := json.Unmarshal(responseBody, &res); err != nil {
g.Log.Fatalf("Failed to unmarshal response: %v", err)
}
return res, nil
}
func main() {
// Флаги для параметров проектов
platformIDFlag := flag.Int("platformid", 0, "Platform ID")
repoIDFlag := flag.Int("repoid", 0, "Repository ID")
ownerFlag := flag.String("owner", "", "Owner name")
cacheTTLFlag := flag.Duration("cachettl", 12*time.Hour, "Cache TTL duration")
forceFlag := flag.Bool("force", false, "Force refresh cache")
cacheFilePathFlag := flag.String("cachefile", "cache.json", "Cache file path")
flag.Parse()
// Загрузка переменных окружения из файла .env
err := godotenv.Load()
if err != nil {
log.Fatalf("Error loading .env file: %v", err)
}
abfURL := os.Getenv("ABF_API_URL")
fileStoreURL := os.Getenv("ABF_FILE_STORE_URL")
login := os.Getenv("ABF_LOGIN")
password := os.Getenv("ABF_PASSWORD")
giteaAPIURL := os.Getenv("GITEA_API_URL")
giteaToken := os.Getenv("GITEA_TOKEN")
giteaOwner := os.Getenv("GITEA_OWNER")
cacheFilePath := *cacheFilePathFlag
// Приоритет флагов над переменными окружения
platformID := *platformIDFlag
if platformID == 0 {
platformIDStr := os.Getenv("PLATFORM_ID")
if platformIDStr == "" {
log.Fatalf("Platform ID is not set either via --platformid flag or PLATFORM_ID environment variable")
}
platformID, err = strconv.Atoi(platformIDStr)
if err != nil {
log.Fatalf("Invalid PLATFORM_ID value: %v", err)
}
}
repoID := *repoIDFlag
if repoID == 0 {
repoIDStr := os.Getenv("REPO_ID")
if repoIDStr == "" {
log.Fatalf("Repository ID is not set either via --repoid flag or REPO_ID environment variable")
}
repoID, err = strconv.Atoi(repoIDStr)
if err != nil {
log.Fatalf("Invalid REPO_ID value: %v", err)
}
}
ownerName := *ownerFlag
if ownerName == "" {
ownerName = os.Getenv("OWNER_NAME")
if ownerName == "" {
log.Fatalf("Owner name is not set either via --owner flag or OWNER_NAME environment variable")
}
}
logger := log.New(os.Stdout, "", log.LstdFlags)
abfClient, err := NewAbfJson(abfURL, fileStoreURL, login, password, logger)
if err != nil {
logger.Fatalf("Failed to create ABF client: %v", err)
}
// Проверка кеша
cacheValid, err := abfClient.IsCacheValid(cacheFilePath, *cacheTTLFlag)
if err != nil {
logger.Fatalf("Failed to check cache validity: %v", err)
}
if cacheValid && !*forceFlag {
logger.Println("Using cached data.")
} else {
logger.Println("Refreshing cache.")
// Получение списка проектов
projects, err := abfClient.GetProjectsList(platformID, repoID, ownerName)
if err != nil {
logger.Fatalf("Failed to get projects: %v", err)
}
logger.Printf("Total projects found: %d", len(projects))
// Сбор ID и имени проектов
projectIDs := make(map[int]string)
for _, project := range projects {
projectIDs[project.ID] = project.Name
}
// Получение деталей проектов и их URL
var projectDetailsList []*ProjectDetails
for projectID, projectName := range projectIDs {
details, err := abfClient.GetProjectDetails(projectID)
if err != nil {
logger.Printf("Failed to get details for project %d (%s): %v", projectID, projectName, err)
continue
}
projectDetailsList = append(projectDetailsList, details)
}
// Сохранение кеша
abfClient.CacheData = make(map[string][]byte)
for _, details := range projectDetailsList {
detailsBytes, err := json.Marshal(details)
if err != nil {
logger.Fatalf("Failed to marshal project details: %v", err)
}
abfClient.CacheData[strconv.Itoa(details.ID)] = detailsBytes
}
if err := abfClient.SaveCache(cacheFilePath); err != nil {
logger.Fatalf("Failed to save cache: %v", err)
}
// Вывод списка проектов с URL
for _, details := range projectDetailsList {
logger.Printf("Project ID: %d, Name: %s, URL: %s", details.ID, details.Name, details.URL)
}
}
// Загрузка кеша для дальнейшей работы
if err := abfClient.LoadCache(cacheFilePath); err != nil {
logger.Fatalf("Failed to load cache: %v", err)
}
// Получение списка репозиториев из Gitea
giteaClient := &GiteaClient{
APIURL: giteaAPIURL,
Token: giteaToken,
Owner: giteaOwner,
Log: logger,
}
giteaRepos, err := giteaClient.GetRepositories()
if err != nil {
logger.Fatalf("Failed to get repositories from Gitea: %v", err)
}
giteaRepoNames := make(map[string]struct{})
for _, repo := range giteaRepos {
giteaRepoNames[repo.Name] = struct{}{}
}
// Миграция репозиториев в Gitea
for projectIDStr, detailsBytes := range abfClient.CacheData {
var projectDetails ProjectDetails
if err := json.Unmarshal(detailsBytes, &projectDetails); err != nil {
logger.Printf("Failed to unmarshal project details: %v", err)
continue
}
projectID, err := strconv.Atoi(projectIDStr)
if err != nil {
logger.Printf("Недопустимый идентификатор проекта %s: %v", projectIDStr, err)
continue
}
// Используйте переменную projectID здесь
logger.Printf("Идентификатор проекта: %d", projectID)
repoName := projectDetails.Name
cloneURL := projectDetails.URL
if _, exists := giteaRepoNames[repoName]; exists {
logger.Printf("Repository %s already exists in Gitea. Skipping...", repoName)
continue
}
err = giteaClient.MirrorRepository(repoName, cloneURL)
if err != nil {
logger.Printf("Failed to mirror repository %s: %v", repoName, err)
} else {
logger.Printf("Repository %s mirrored to Gitea.", repoName)
}
}
}