package main import ( "bytes" "encoding/base64" "encoding/json" "flag" "fmt" "io/ioutil" "log" "net/http" "os" "strconv" "strings" "time" "github.com/joho/godotenv" ) // AbfJson структура для взаимодействия с API ABF.io type AbfJson struct { AbfURL string FileStoreURL string Login string Password string Base64AuthString string Log *log.Logger CacheData map[string][]byte CacheEtags map[string]string } // Ошибки API ABF.io var errorsMap = map[string]error{ "Invalid email or password.": fmt.Errorf("Authentication error"), "403 Forbidden | Rate Limit Exceeded": fmt.Errorf("Rate limit exceeded"), "Page not found": fmt.Errorf("Page not found"), "Project has not been forked. Name has already been taken": fmt.Errorf("Name already taken"), "Error 404. Resource not found!": fmt.Errorf("Resource not found"), "Something went wrong. We've been notified about this issue and we'll take a look at it shortly.": fmt.Errorf("Internal server error"), "We update the site, it will take some time. We are really trying to do it fast. We apologize for any inconvenience..": fmt.Errorf("Server under maintenance"), "Requires authentication": fmt.Errorf("Authentication required"), "Forbidden. Sorry, you don't have enough rights for this action!": fmt.Errorf("Forbidden"), "Access violation to this page!": fmt.Errorf("Access violation"), "Bad Request": fmt.Errorf("Bad request"), } // GoodMessages сообщения, которые не считаются ошибками var goodMessages = []string{ "Errors during build publishing!", "Build is queued for publishing", } // FatalErrors фатальные ошибки, которые приводят к выходу из программы var fatalErrors = []error{ errorsMap["Invalid email or password."], errorsMap["403 Forbidden | Rate Limit Exceeded"], errorsMap["Something went wrong. We've been notified about this issue and we'll take a look at it shortly."], } // Project структура для хранения информации о проекте type Project struct { ID int `json:"id"` Name string `json:"name"` Description string `json:"description"` } // ProjectDetails структура для хранения деталей проекта type ProjectDetails struct { ID int `json:"id"` Name string `json:"name"` URL string `json:"git_url"` WebURL string `json:"web_url"` SSHURLToRepo string `json:"ssh_url_to_repo"` HTTPURLToRepo string `json:"http_url_to_repo"` } // CacheEntry структура для хранения данных кеша type CacheEntry struct { Data map[string][]byte `json:"data"` CacheEtags map[string]string `json:"cache_etags"` Timestamp time.Time `json:"timestamp"` } // NewAbfJson создает новый экземпляр AbfJson func NewAbfJson(abfURL, fileStoreURL, login, password string, log *log.Logger) (*AbfJson, error) { if !strings.HasPrefix(fileStoreURL, "http://") && !strings.HasPrefix(fileStoreURL, "https://") { log.Fatalf("File-store URL has to start with 'http(s)://'") } lpw := fmt.Sprintf("%s:%s", login, password) encodedLpw := base64.StdEncoding.EncodeToString([]byte(lpw)) return &AbfJson{ AbfURL: strings.TrimSuffix(abfURL, "/"), FileStoreURL: strings.TrimSuffix(fileStoreURL, "/"), Login: login, Password: password, Base64AuthString: encodedLpw, Log: log, CacheData: make(map[string][]byte), CacheEtags: make(map[string]string), }, nil } // LoadCache загружает кеш из файла func (a *AbfJson) LoadCache(cacheFilePath string) error { data, err := ioutil.ReadFile(cacheFilePath) if err != nil { if os.IsNotExist(err) { a.Log.Println("Cache file does not exist. Proceeding without cache.") return nil } return fmt.Errorf("failed to read cache file: %v", err) } var cacheEntry CacheEntry if err := json.Unmarshal(data, &cacheEntry); err != nil { return fmt.Errorf("failed to unmarshal cache data: %v", err) } a.CacheData = cacheEntry.Data a.CacheEtags = cacheEntry.CacheEtags a.Log.Println("Cache loaded successfully.") return nil } // SaveCache сохраняет кеш в файл func (a *AbfJson) SaveCache(cacheFilePath string) error { cacheEntry := CacheEntry{ Data: a.CacheData, CacheEtags: a.CacheEtags, Timestamp: time.Now(), } data, err := json.Marshal(cacheEntry) if err != nil { return fmt.Errorf("failed to marshal cache data: %v", err) } if err := ioutil.WriteFile(cacheFilePath, data, 0644); err != nil { return fmt.Errorf("failed to write cache file: %v", err) } a.Log.Println("Cache saved successfully.") return nil } // IsCacheValid проверяет, действителен ли кеш func (a *AbfJson) IsCacheValid(cacheFilePath string, ttl time.Duration) (bool, error) { data, err := ioutil.ReadFile(cacheFilePath) if err != nil { if os.IsNotExist(err) { a.Log.Println("Cache file does not exist. Cache is invalid.") return false, nil } return false, fmt.Errorf("failed to read cache file: %v", err) } var cacheEntry CacheEntry if err := json.Unmarshal(data, &cacheEntry); err != nil { return false, fmt.Errorf("failed to unmarshal cache data: %v", err) } if time.Since(cacheEntry.Timestamp) > ttl { a.Log.Println("Cache has expired.") return false, nil } a.Log.Println("Cache is valid.") return true, nil } // ProcessResponse обрабатывает ответ API ABF.io func (a *AbfJson) ProcessResponse(responseString []byte) (map[string]interface{}, error) { var res map[string]interface{} if err := json.Unmarshal(responseString, &res); err != nil { a.Log.Fatalf("Internal server error: it has returned non-json data.") } message, ok := res["message"].(string) if ok && !contains(goodMessages, message) { return nil, errorsMap[message] } repository, repoOk := res["repository"].(map[string]interface{}) project, projOk := res["project"].(map[string]interface{}) if repoOk { repoMessage, ok := repository["message"].(string) if ok && (strings.Contains(repoMessage, "error") || strings.Contains(repoMessage, "has not been")) { return nil, errorsMap[repoMessage] } } if projOk { projMessage, ok := project["message"].(string) if ok && (strings.Contains(projMessage, "error") || strings.Contains(projMessage, "has not been")) { return nil, errorsMap[projMessage] } } if errStr, ok := res["error"].(string); ok && !contains(goodMessages, errStr) { return nil, errorsMap[errStr] } return res, nil } // GetURLContents выполняет HTTP-запрос к API ABF.io func (a *AbfJson) GetURLContents(path string, params map[string]string, method string, body interface{}) (interface{}, error) { url := a.AbfURL + path if method == "file_store" { url = a.FileStoreURL + path } if len(params) > 0 { query := "" for key, value := range params { if query == "" { query += "?" } else { query += "&" } query += fmt.Sprintf("%s=%s", key, value) } url += query } a.Log.Printf("Fetching URL: %s", url) var req *http.Request var err error var requestBody []byte if body != nil { requestBody, err = json.Marshal(body) if err != nil { a.Log.Fatalf("Failed to marshal request body: %v", err) } var prettyJSON bytes.Buffer json.Indent(&prettyJSON, requestBody, "", " ") a.Log.Printf("Request Body: %s", prettyJSON.String()) // Логирование отправляемого JSON } switch method { case "POST": req, err = http.NewRequest("POST", url, bytes.NewBuffer(requestBody)) case "PUT": req, err = http.NewRequest("PUT", url, bytes.NewBuffer(requestBody)) case "DELETE": req, err = http.NewRequest("DELETE", url, bytes.NewBuffer(requestBody)) default: req, err = http.NewRequest("GET", url, nil) } if err != nil { a.Log.Fatalf("Failed to create request: %v", err) } req.Header.Add("Authorization", fmt.Sprintf("Basic %s", a.Base64AuthString)) req.Header.Add("Content-Type", "application/json") etag, etagFound := a.CacheEtags[url] if etagFound { req.Header.Add("If-None-Match", etag) } client := &http.Client{ Timeout: 30 * time.Second, } resp, err := client.Do(req) if err != nil { a.Log.Fatalf("HTTP request failed: %v", err) } defer resp.Body.Close() responseBody, err := ioutil.ReadAll(resp.Body) if err != nil { a.Log.Fatalf("Failed to read response body: %v", err) } if resp.StatusCode == 304 && etagFound { a.Log.Println("Getting cached result (cache was validated)") return a.ProcessResponse(a.CacheData[etag]) } if resp.StatusCode != 200 && resp.StatusCode != 201 { a.Log.Printf("Return code: %d", resp.StatusCode) return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(responseBody)) } etagNew := resp.Header.Get("ETag") if etagNew != "" { a.Log.Printf("Caching the new value for %s. ETag is %s", url, etagNew) a.CacheEtags[url] = etagNew a.CacheData[etagNew] = responseBody } var res interface{} if err := json.Unmarshal(responseBody, &res); err != nil { a.Log.Fatalf("Failed to unmarshal response: %v", err) } return res, nil } // GetProjectsList получает список проектов с обходом всех страниц func (a *AbfJson) GetProjectsList(platformID, repoID int, ownerName string) ([]Project, error) { var allProjects []Project page := 1 perPage := 100 for { params := map[string]string{ "added": "true", "format": "json", "owner_name": ownerName, "page": fmt.Sprintf("%d", page), "per_page": fmt.Sprintf("%d", perPage), "project_name": "", } path := fmt.Sprintf("/platforms/%d/repositories/%d/projects_list", platformID, repoID) res, err := a.GetURLContents(path, params, "GET", nil) if err != nil { return nil, fmt.Errorf("Failed to get projects list: %v", err) } // Преобразуем res["projects"] в []byte projectsArray, ok := res.(map[string]interface{})["projects"] if !ok { a.Log.Fatalf("Failed to find 'projects' in response") } projectsBytes, err := json.Marshal(projectsArray) if err != nil { a.Log.Fatalf("Failed to marshal projects list: %v", err) } var projects []Project if err := json.Unmarshal(projectsBytes, &projects); err != nil { a.Log.Fatalf("Failed to unmarshal projects list: %v", err) } allProjects = append(allProjects, projects...) if len(projects) < perPage { break } page++ } return allProjects, nil } // GetProjectDetails получает детали проекта по ID func (a *AbfJson) GetProjectDetails(projectID int) (*ProjectDetails, error) { path := fmt.Sprintf("/api/v1/projects/%d.json", projectID) res, err := a.GetURLContents(path, nil, "GET", nil) if err != nil { a.Log.Printf("Failed to get project details for project ID %d: %v", projectID, err) return nil, err } projectData, ok := res.(map[string]interface{})["project"].(map[string]interface{}) if !ok { a.Log.Printf("Missing 'project' field in response for project ID %d", projectID) return nil, fmt.Errorf("Missing 'project' field in response") } id, idOk := projectData["id"] if !idOk { a.Log.Printf("Missing 'id' field in project details for project ID %d", projectID) return nil, fmt.Errorf("Missing 'id' field in project details") } idFloat, idOk := id.(float64) if !idOk { a.Log.Printf("Invalid 'id' type in project details for project ID %d: expected float64, got %T", projectID, id) return nil, fmt.Errorf("Invalid 'id' type in project details") } name, nameOk := projectData["name"].(string) if !nameOk { a.Log.Printf("Missing 'name' field in project details for project ID %d", projectID) return nil, fmt.Errorf("Missing 'name' field in project details") } gitUrl, gitUrlOk := projectData["git_url"].(string) webUrl, webUrlOk := projectData["web_url"].(string) sshUrlToRepo, sshUrlToRepoOk := projectData["ssh_url_to_repo"].(string) httpUrlToRepo, httpUrlToRepoOk := projectData["http_url_to_repo"].(string) var url string if gitUrlOk { url = gitUrl } else if httpUrlToRepoOk { url = httpUrlToRepo } else if webUrlOk { url = webUrl } else if sshUrlToRepoOk { url = sshUrlToRepo } else { a.Log.Printf("Missing URL fields in project details for project ID %d", projectID) return nil, fmt.Errorf("Missing URL fields in project details") } projectDetails := &ProjectDetails{ ID: int(idFloat), Name: name, URL: url, WebURL: webUrl, SSHURLToRepo: sshUrlToRepo, HTTPURLToRepo: httpUrlToRepo, } return projectDetails, nil } // contains проверяет, содержит ли слайс строк строку func contains(slice []string, str string) bool { for _, item := range slice { if item == str { return true } } return false } // GiteaClient структура для взаимодействия с API Gitea type GiteaClient struct { APIURL string Token string Owner string Log *log.Logger } // Repository структура для хранения информации о репозитории в Gitea type Repository struct { ID int `json:"id"` Name string `json:"name"` } // GetRepositories получает список репозиториев из Gitea func (g *GiteaClient) GetRepositories() ([]Repository, error) { path := "/user/repos" res, err := g.GetURLContents(path, nil, "GET", nil) if err != nil { return nil, fmt.Errorf("Failed to get repositories: %v", err) } // Проверяем, что res является массивом reposArray, ok := res.([]interface{}) if !ok { g.Log.Fatalf("Expected array in response, got %T", res) } var repos []Repository for _, repoInterface := range reposArray { repoMap, ok := repoInterface.(map[string]interface{}) if !ok { g.Log.Fatalf("Expected map in array element, got %T", repoInterface) } id, idOk := repoMap["id"].(float64) if !idOk { g.Log.Fatalf("Invalid 'id' type in repository: expected float64, got %T", repoMap["id"]) } name, nameOk := repoMap["name"].(string) if !nameOk { g.Log.Fatalf("Missing 'name' field in repository") } repos = append(repos, Repository{ ID: int(id), Name: name, }) } return repos, nil } // MirrorRepositoryPayload структура для запроса зеркалирования репозитория в Gitea type MirrorRepositoryPayload struct { AuthPassword string `json:"auth_password,omitempty"` AuthToken string `json:"auth_token,omitempty"` AuthUsername string `json:"auth_username,omitempty"` CloneAddr string `json:"clone_addr"` Description string `json:"description,omitempty"` Issues bool `json:"issues"` Labels bool `json:"labels"` LFS bool `json:"lfs"` LFSEndpoint string `json:"lfs_endpoint,omitempty"` Milestones bool `json:"milestones"` Mirror bool `json:"mirror"` MirrorInterval string `json:"mirror_interval,omitempty"` Private bool `json:"private"` PullRequests bool `json:"pull_requests"` Releases bool `json:"releases"` RepoName string `json:"repo_name"` RepoOwner string `json:"repo_owner"` Service string `json:"service"` UID int `json:"uid,omitempty"` Wiki bool `json:"wiki"` } // MirrorRepository создает зеркало репозитория в Gitea func (g *GiteaClient) MirrorRepository(repoName, cloneURL string) error { payload := MirrorRepositoryPayload{ CloneAddr: cloneURL, Description: "", Issues: true, Labels: true, LFS: false, Milestones: true, Mirror: true, MirrorInterval: "8h0m0s", // Интервал зеркалирования Private: false, PullRequests: true, Releases: true, RepoName: repoName, RepoOwner: g.Owner, Service: "git", Wiki: true, } path := "/repos/migrate" res, err := g.GetURLContents(path, nil, "POST", payload) if err != nil { return fmt.Errorf("Failed to mirror repository: %v", err) } if res == nil { return fmt.Errorf("Empty response from Gitea") } g.Log.Printf("Response from Gitea: %+v", res) return nil } // GetURLContents выполняет HTTP-запрос к API Gitea func (g *GiteaClient) GetURLContents(path string, params map[string]string, method string, body interface{}) (interface{}, error) { url := g.APIURL + path if len(params) > 0 { query := "" for key, value := range params { if query == "" { query += "?" } else { query += "&" } query += fmt.Sprintf("%s=%s", key, value) } url += query } g.Log.Printf("Fetching URL: %s", url) var req *http.Request var err error var requestBody []byte if body != nil { requestBody, err = json.Marshal(body) if err != nil { g.Log.Fatalf("Failed to marshal request body: %v", err) } var prettyJSON bytes.Buffer json.Indent(&prettyJSON, requestBody, "", " ") g.Log.Printf("Request Body: %s", prettyJSON.String()) // Логирование отправляемого JSON } switch method { case "POST": req, err = http.NewRequest("POST", url, bytes.NewBuffer(requestBody)) case "PUT": req, err = http.NewRequest("PUT", url, bytes.NewBuffer(requestBody)) case "DELETE": req, err = http.NewRequest("DELETE", url, bytes.NewBuffer(requestBody)) default: req, err = http.NewRequest("GET", url, nil) } if err != nil { g.Log.Fatalf("Failed to create request: %v", err) } req.Header.Add("Authorization", fmt.Sprintf("token %s", g.Token)) req.Header.Add("Content-Type", "application/json") client := &http.Client{ Timeout: 30 * time.Second, } resp, err := client.Do(req) if err != nil { g.Log.Fatalf("HTTP request failed: %v", err) } defer resp.Body.Close() responseBody, err := ioutil.ReadAll(resp.Body) if err != nil { g.Log.Fatalf("Failed to read response body: %v", err) } if resp.StatusCode != 200 && resp.StatusCode != 201 { g.Log.Printf("Return code: %d", resp.StatusCode) return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(responseBody)) } var res interface{} if err := json.Unmarshal(responseBody, &res); err != nil { g.Log.Fatalf("Failed to unmarshal response: %v", err) } return res, nil } func main() { // Флаги для параметров проектов platformIDFlag := flag.Int("platformid", 0, "Platform ID") repoIDFlag := flag.Int("repoid", 0, "Repository ID") ownerFlag := flag.String("owner", "", "Owner name") cacheTTLFlag := flag.Duration("cachettl", 12*time.Hour, "Cache TTL duration") forceFlag := flag.Bool("force", false, "Force refresh cache") cacheFilePathFlag := flag.String("cachefile", "cache.json", "Cache file path") flag.Parse() // Загрузка переменных окружения из файла .env err := godotenv.Load() if err != nil { log.Fatalf("Error loading .env file: %v", err) } abfURL := os.Getenv("ABF_API_URL") fileStoreURL := os.Getenv("ABF_FILE_STORE_URL") login := os.Getenv("ABF_LOGIN") password := os.Getenv("ABF_PASSWORD") giteaAPIURL := os.Getenv("GITEA_API_URL") giteaToken := os.Getenv("GITEA_TOKEN") giteaOwner := os.Getenv("GITEA_OWNER") cacheFilePath := *cacheFilePathFlag // Приоритет флагов над переменными окружения platformID := *platformIDFlag if platformID == 0 { platformIDStr := os.Getenv("PLATFORM_ID") if platformIDStr == "" { log.Fatalf("Platform ID is not set either via --platformid flag or PLATFORM_ID environment variable") } platformID, err = strconv.Atoi(platformIDStr) if err != nil { log.Fatalf("Invalid PLATFORM_ID value: %v", err) } } repoID := *repoIDFlag if repoID == 0 { repoIDStr := os.Getenv("REPO_ID") if repoIDStr == "" { log.Fatalf("Repository ID is not set either via --repoid flag or REPO_ID environment variable") } repoID, err = strconv.Atoi(repoIDStr) if err != nil { log.Fatalf("Invalid REPO_ID value: %v", err) } } ownerName := *ownerFlag if ownerName == "" { ownerName = os.Getenv("OWNER_NAME") if ownerName == "" { log.Fatalf("Owner name is not set either via --owner flag or OWNER_NAME environment variable") } } logger := log.New(os.Stdout, "", log.LstdFlags) abfClient, err := NewAbfJson(abfURL, fileStoreURL, login, password, logger) if err != nil { logger.Fatalf("Failed to create ABF client: %v", err) } // Проверка кеша cacheValid, err := abfClient.IsCacheValid(cacheFilePath, *cacheTTLFlag) if err != nil { logger.Fatalf("Failed to check cache validity: %v", err) } if cacheValid && !*forceFlag { logger.Println("Using cached data.") } else { logger.Println("Refreshing cache.") // Получение списка проектов projects, err := abfClient.GetProjectsList(platformID, repoID, ownerName) if err != nil { logger.Fatalf("Failed to get projects: %v", err) } logger.Printf("Total projects found: %d", len(projects)) // Сбор ID и имени проектов projectIDs := make(map[int]string) for _, project := range projects { projectIDs[project.ID] = project.Name } // Получение деталей проектов и их URL var projectDetailsList []*ProjectDetails for projectID, projectName := range projectIDs { details, err := abfClient.GetProjectDetails(projectID) if err != nil { logger.Printf("Failed to get details for project %d (%s): %v", projectID, projectName, err) continue } projectDetailsList = append(projectDetailsList, details) } // Сохранение кеша abfClient.CacheData = make(map[string][]byte) for _, details := range projectDetailsList { detailsBytes, err := json.Marshal(details) if err != nil { logger.Fatalf("Failed to marshal project details: %v", err) } abfClient.CacheData[strconv.Itoa(details.ID)] = detailsBytes } if err := abfClient.SaveCache(cacheFilePath); err != nil { logger.Fatalf("Failed to save cache: %v", err) } // Вывод списка проектов с URL for _, details := range projectDetailsList { logger.Printf("Project ID: %d, Name: %s, URL: %s", details.ID, details.Name, details.URL) } } // Загрузка кеша для дальнейшей работы if err := abfClient.LoadCache(cacheFilePath); err != nil { logger.Fatalf("Failed to load cache: %v", err) } // Получение списка репозиториев из Gitea giteaClient := &GiteaClient{ APIURL: giteaAPIURL, Token: giteaToken, Owner: giteaOwner, Log: logger, } giteaRepos, err := giteaClient.GetRepositories() if err != nil { logger.Fatalf("Failed to get repositories from Gitea: %v", err) } giteaRepoNames := make(map[string]struct{}) for _, repo := range giteaRepos { giteaRepoNames[repo.Name] = struct{}{} } // Миграция репозиториев в Gitea for projectIDStr, detailsBytes := range abfClient.CacheData { var projectDetails ProjectDetails if err := json.Unmarshal(detailsBytes, &projectDetails); err != nil { logger.Printf("Failed to unmarshal project details: %v", err) continue } projectID, err := strconv.Atoi(projectIDStr) if err != nil { logger.Printf("Недопустимый идентификатор проекта %s: %v", projectIDStr, err) continue } // Используйте переменную projectID здесь logger.Printf("Идентификатор проекта: %d", projectID) repoName := projectDetails.Name cloneURL := projectDetails.URL if _, exists := giteaRepoNames[repoName]; exists { logger.Printf("Repository %s already exists in Gitea. Skipping...", repoName) continue } err = giteaClient.MirrorRepository(repoName, cloneURL) if err != nil { logger.Printf("Failed to mirror repository %s: %v", repoName, err) } else { logger.Printf("Repository %s mirrored to Gitea.", repoName) } } }