Hi @Ara4Sh
# coding: utf-8
import json, re, requests, urllib, copy, sys
import collections
from pprint import pprint
newheaders = {'content-type': 'application/json', "PRIVATE-TOKEN": "mynewtoken", "Accept": "application/json" }
oldheaders = {'content-type': 'application/json', "PRIVATE-TOKEN": "myoldtoken", "Accept": "application/json" }
oldapi='https://myoldgitlab/api/v3/'
newapi='https://mynewgitlab.com/api/v3/'
grouplen= # value can be found here : https://mygitlab/admin
nbproj= # value can be found here : https://mygitlab/admin
nbusr= # value can be found here : https://mygitlab/admin
'''
I don't use http://doc.gitlab.com/ce/api/groups.html#list-groups nor
http://doc.gitlab.com/ce/api/projects.html#list-all-projects in this
code. Those routes are paginated to return max 20 results. To avoid
pagination handling, I chose to grunt list all the things :)
Long story short: its necessary to set grouplen, nbproj, nbusr.
'''
'''
NOTA BENE :
I highly recommend that you try this script on a mock server first,
and then tweak it at your will. If you add some features,
I bet that the community will love them so please share ;-).
'''
def bruteforcegroups(api=oldapi, h=oldheaders):
i=0
groups=[]
if api==oldapi:
# while len(groups)<grouplen:
while len(groups)!=grouplen:
url='%sgroups/%s' % (api, str(i))
r = requests.get(url, headers=h)
i=i+1
if 'name' in r.json():
groups.append(dict(r.json()))
return groups
if api==newapi:
# while len(groups)<(grouplen): # Here was a -1
while len(groups)!=(grouplen):
url='%sgroups/%s' % (api, str(i))
r = requests.get(url, headers=h)
i=i+1
if 'name' in r.json():
groups.append(dict(r.json()))
return groups
def creategroups(groups):
responses=[]
headers=newheaders
for group in groups:
data={'name': group['name'],
'path': group['path'],
'description': group['description']
}
url='%sgroups' % (newapi)
data=json.dumps(data)
r=requests.post(url, data, headers=headers)
responses.append(r.json())
for resp in responses: # TODO put this in logger
print resp
def listprojects(api=oldapi, h=oldheaders):
proj=[]
trash=[]
i=0
while len(proj) != nbproj:
print i
url='%sprojects/%s' % (api, str(i))
r = requests.get(url, headers=h)
i=i+1
if 'name' in dict(r.json()):
proj.append(dict(r.json()))
print 'found ', r.json()['name']
print len(proj), "projects found"
print len(proj), "projects to be created"
return proj
def createprojects(newgroups, groups, g):
responses=[]
proj=listprojects()
default={
"name":"value",
"namespace_id":"",
"description":"Default",
"issues_enabled":"True",
"merge_requests_enabled":"True",
"builds_enabled":"True",
"wiki_enabled":"True",
"snippets_enabled":"True",
"public":"False",
"visibility_level":"Private",
"public_builds":"False"
}
url=newapi
headers=newheaders
url='%sprojects' % (url)
for project in proj:
new=copy.deepcopy(default)
new['name']=project['name']
new['description']=project['description']
new['namespace_id']=getnewgroupid(project['namespace']['name'], g)
print new['namespace_id'], project['namespace']['id'], project['namespace']['name']
data=json.dumps(new)
r=requests.post(url, data, headers=headers)
print r.json()
responses.append(r.json())
return
def getnewgroupid(groupname, g):
for group in g:
if group == groupname:
if 'new' in g[group]:
return g[group]['new']
else:
return '1'
def getmynamespace(merger, nsid):
nsid=str(nsid)
for group in merger:
if str(merger[group]['old']) == nsid:
if 'new' not in merger[group]:
print '%s seems to have a problem' % str(group) #TODO Logger
return False
else:
return merger[group]['new']
def diffgroups(groups, newgroups):
temp=[]
temp2=[]
pouet=[]
for group in groups:
temp.append(group['name'])
for group in newgroups:
temp2.append(group['name'])
diff=list(set(temp) - set(temp2))
for i in diff:
for n in groups:
if str(i) in n['name']:
for proj in n:
pouet.append(proj)
print len(proj), "missing projects"
return
def listusers(api=oldapi, h=oldheaders):
toret=[]
i=0
while len(toret) != nbusr:
print i
url='%susers/%s' % (api, str(i))
r = requests.get(url, headers=h)
if 'name' in r.json():
print r.json()['name'], "has been found"
url='%susers/%s/keys' % (api, str(i))
rr=requests.get(url, headers=h)
keys={}
w=0
for key in rr.json():
keys[w]=dict(key)
w=w+1
r=dict(r.json())
r['ssh']=keys
toret.append(r)
i=i+1
return toret
def createusers(users, api=newapi, h=newheaders):
uresponses=[]
kresponses=[]
'''
http://doc.gitlab.com/ce/api/users.html#user-creation
http://doc.gitlab.com/ce/api/users.html#add-ssh-key-for-user
'''
defaultuser={
'email': '', # Mandatory
'password': 'CHANGEME', # Mandatory
'username': '', # Mandatory
'name': '', # Mandatory
'skype': '', # optionnal
'linkedin': '', # optionnal
'twitter': '', # optionnal
'website_url': '', # optionnal
'projects_limit': '1000', # optionnal
'extern_uid': '', # optionnal
'provider': '', # optionnal
'bio': '', # optionnal
'admin': '', # optionnal
'can_create_group': '', # optionnal
'confirm': 'False' # optionnal
}
defaultkey={ # All mandatory
'id':'',
'title':'',
'key':''
}
for user in users:
print user['name'], 'will be created'
new=copy.deepcopy(defaultuser)
ssh=[]
for key, value in user.iteritems():
if key not in ['projects_limit']:
new[key]=value
url='%susers' % (api)
data=json.dumps(new)
r=requests.post(url, data, headers=h)
uresponses.append(dict(r.json()))
for key, value in user.iteritems():
if key in ['ssh']:
for k,v in value.iteritems():
userkey=copy.deepcopy(defaultkey)
for i,j in v.iteritems():
if i in defaultkey:
userkey[i]=j
data=json.dumps(userkey)
uid=user['id']
url='%s/%s/keys'%(url, uid)
r=requests.post(url, data, headers=h)
for i in uresponses: # TODO LOGGER
print i # TODO LOGGER
for i in kresponses: # TODO LOGGER
print i # TODO LOGGER
def mirrorgroupmembers(group, usermatch, groupmatch, api=oldapi, h=oldheaders):
resp=[]
''' http://doc.gitlab.com/ce/api/groups.html#list-group-members '''
userscheme={
'id':'',
'user_id':'',
'access_level':''
}
url='%sgroups/%s/members' % (api, group)
r=requests.get(url, headers=h)
r=r.json()
for m in r:
user=copy.deepcopy(userscheme)
user['id']=getgroupid(group, groupmatch)
user['user_id']=getuserid(m['id'], usermatch, groupmatch)
user['access_level']=m['access_level']
data=json.dumps(user)
# print data
createurl='%sgroups/%s/members' % (newapi, user['id'])
create=requests.post(createurl, data, headers=newheaders)
create=create.json()
resp.append(create)
for i in resp:
print i
def getgroupid(groupid, groups):
for group in groups:
if str(groups[group]['old'])==str(groupid):
if 'new' in groups[group]:
return groups[group]['new']
print group, groups[group]['old'], groups[group]['new']
else:
print 'There is no matching new group for %s ' % group
return group
def getuserid(userid, users, groups):
for user in users:
if str(users[user]['old'])==str(userid):
if 'new' in users[user]:
return users[user]['new']
else:
print "no new id for user %s" % user
for u in listusers(api=newapi, h=newheaders):
if u['name'] == user:
return u['id']
print 'No user has beend found for %s' % user
def usermapper(users, newusers):
u={}
for user in users:
if user['name'] not in u:
u[user['name']]={}
u[user['name']]['old']=user['id']
for user in newusers:
if user['name'] not in u:
pass # Only with administrator # TODO logger
else:
u[user['name']]['new']=user['id']
return u
def groupmapper(groups, newgroups):
g={}
for group in groups:
if group['name'] not in g:
g[group['name']]={}
g[group['name']]['old']=group['id']
for group in newgroups:
if group['name'] not in g:
print 'problem with ', group['name']
else:
g[group['name']]['new']=group['id']
return g
def mirrorgit():
import os
for p in listprojects():
URL = p['ssh_url_to_repo']
NURL= re.sub('oldgitlab', 'newgitlab', URL)
os.system('bash tool.sh %s %s' % (URL, NURL))
def createadmin(groups, g, u, api=newapi, h=newheaders):
''' http://doc.gitlab.com/ce/api/groups.html '''
userscheme={
'id':'',
'user_id':'',
'access_level':'50'
}
for group in groups:
user=copy.deepcopy(userscheme)
user['id']=getgroupid(group['id'], g)
user['user_id']=getadminid(u)
createurl='%sgroups/%s/members' % (api, user['id'])
data=json.dumps(user)
create=requests.post(createurl, data, headers=h)
print create.json() # TODO logger
def getadminid(u):
for user in u:
if user == 'administrator':
return u[user]['new']
def unprotect(projects, api=oldapi, headers=oldheaders):
for p in projects:
branches=listbranches(p['id'], api, headers)
print p['id'], p['name']
for branch in branches:
print branch['name']
url="%sprojects/%s/repository/branches/%s/unprotect" % (api, p['id'], branch['name'])
r=requests.put(url, headers=headers)
print r.json()
print "Unprotected ", branch
def listbranches(projectid, api, headers):
url="%sprojects/%s/repository/branches" % (api, projectid)
r=requests.get(url, headers=headers)
return r.json()
def main():
groups=bruteforcegroups()
creategroups(groups)
newgroups=bruteforcegroups(api=newapi, h=newheaders)
g=groupmapper(groups, newgroups)
users=listusers()
createusers(users)
newusers=listusers(api=newapi, h=newheaders)
u=usermapper(users, newusers)
createprojects(newgroups, groups, g)
for group in groups:
mirrorgroupmembers(group['id'], u, g)
createadmin(groups, g, u)
proj=listprojects(api=newapi, h=newheaders)
unprotect(proj)
mirrorgit()
main()
and here is tool.sh
echo -e "\e[31mCHANGE MYNEWGITLAB BY YOUR NEW GITLAB IN THIS SCRIPT\e[0m"
echo -e "\e[31mCHANGE MYNEWGITLAB BY YOUR NEW GITLAB IN THIS SCRIPT\e[0m"
echo -e "\e[31mCHANGE MYNEWGITLAB BY YOUR NEW GITLAB IN THIS SCRIPT\e[0m"
echo -e "\e[31mCHANGE MYNEWGITLAB BY YOUR NEW GITLAB IN THIS SCRIPT\e[0m"
sleep 10
echo $@
cd /tmp
rm -rf "git"
mkdir "git"
cd "git"
git clone --mirror "${1}" .
git remote add gitlab "${2}"
git push -f gitlab refs/heads/*:refs/heads/*
if [ $? -ne 0 ]; then
echo "Checking if this is an empty repo"
git clone "${1}" gito 2>&1 >/dev/null | grep -iq empty
if [ $? -eq 0 ]; then
echo "It appears indeed that this repo is empty, lets give up!"
return 0
fi
fi
if [ $? -ne 0 ]; then
echo "${2} has failed on basic push"
var=$(grep -i "url = git@mynewgitlab" config|awk -F '/' '{ print $2 }'|tr '[:upper:]' '[:lower:]')
sed -i -e "s,/.*,/$var,g" config
git push -f gitlab refs/heads/*:refs/heads/*
if [ $? -ne 0 ]; then
echo "${2} has failed on lowercase push"
sed -i -e "s,git@mynewgitlab.*,git@mynewgitlab:root/$(echo ${2}|awk -F '/' '{ print $2 }'),g" config
git push -f gitlab refs/heads/*:refs/heads/*
if [ $? -ne 0 ]; then
echo "${2} has failed on root and lowercase push"
mv "/tmp/git" "/tmp/git+$(date +%s)"
fi
fi
fi
please note that the script is not usable as is and that I can’t stress enought that you read it entirely and then edit it according to your needs. There is some obvious missing features (no push of tags for instance, all python instead of python + bash, etc.), if you add them : can you please update this thread so that community will benefit from it?
Thank you