mirror of
https://github.com/licsber/micropython.git
synced 2024-09-20 09:00:23 +08:00
tools, unix: Replace upip tarball with just source files.
To make its inclusion as frozen modules in multiple ports less magic. Ports are just expected to symlink 2 files into their scripts/modules subdirs. Unix port updated to use this and in general follow frozen modules setup tested and tried on baremetal ports, where there's "scripts" predefined dir (overridable with FROZEN_DIR make var), and a user just drops Python files there.
This commit is contained in:
parent
bc4441afa7
commit
61d74fdef8
Binary file not shown.
288
tools/upip.py
Normal file
288
tools/upip.py
Normal file
@ -0,0 +1,288 @@
|
|||||||
|
import sys
|
||||||
|
import gc
|
||||||
|
import uos as os
|
||||||
|
import uerrno as errno
|
||||||
|
import ujson as json
|
||||||
|
import uzlib
|
||||||
|
import upip_utarfile as tarfile
|
||||||
|
gc.collect()
|
||||||
|
|
||||||
|
|
||||||
|
debug = False
|
||||||
|
install_path = None
|
||||||
|
cleanup_files = []
|
||||||
|
gzdict_sz = 16 + 15
|
||||||
|
|
||||||
|
file_buf = bytearray(512)
|
||||||
|
|
||||||
|
class NotFoundError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def op_split(path):
|
||||||
|
if path == "":
|
||||||
|
return ("", "")
|
||||||
|
r = path.rsplit("/", 1)
|
||||||
|
if len(r) == 1:
|
||||||
|
return ("", path)
|
||||||
|
head = r[0]
|
||||||
|
if not head:
|
||||||
|
head = "/"
|
||||||
|
return (head, r[1])
|
||||||
|
|
||||||
|
def op_basename(path):
|
||||||
|
return op_split(path)[1]
|
||||||
|
|
||||||
|
# Expects *file* name
|
||||||
|
def _makedirs(name, mode=0o777):
|
||||||
|
ret = False
|
||||||
|
s = ""
|
||||||
|
comps = name.rstrip("/").split("/")[:-1]
|
||||||
|
if comps[0] == "":
|
||||||
|
s = "/"
|
||||||
|
for c in comps:
|
||||||
|
if s and s[-1] != "/":
|
||||||
|
s += "/"
|
||||||
|
s += c
|
||||||
|
try:
|
||||||
|
os.mkdir(s)
|
||||||
|
ret = True
|
||||||
|
except OSError as e:
|
||||||
|
if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR:
|
||||||
|
raise
|
||||||
|
ret = False
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
def save_file(fname, subf):
|
||||||
|
global file_buf
|
||||||
|
with open(fname, "wb") as outf:
|
||||||
|
while True:
|
||||||
|
sz = subf.readinto(file_buf)
|
||||||
|
if not sz:
|
||||||
|
break
|
||||||
|
outf.write(file_buf, sz)
|
||||||
|
|
||||||
|
def install_tar(f, prefix):
|
||||||
|
meta = {}
|
||||||
|
for info in f:
|
||||||
|
#print(info)
|
||||||
|
fname = info.name
|
||||||
|
try:
|
||||||
|
fname = fname[fname.index("/") + 1:]
|
||||||
|
except ValueError:
|
||||||
|
fname = ""
|
||||||
|
|
||||||
|
save = True
|
||||||
|
for p in ("setup.", "PKG-INFO", "README"):
|
||||||
|
#print(fname, p)
|
||||||
|
if fname.startswith(p) or ".egg-info" in fname:
|
||||||
|
if fname.endswith("/requires.txt"):
|
||||||
|
meta["deps"] = f.extractfile(info).read()
|
||||||
|
save = False
|
||||||
|
if debug:
|
||||||
|
print("Skipping", fname)
|
||||||
|
break
|
||||||
|
|
||||||
|
if save:
|
||||||
|
outfname = prefix + fname
|
||||||
|
if info.type != tarfile.DIRTYPE:
|
||||||
|
if debug:
|
||||||
|
print("Extracting " + outfname)
|
||||||
|
_makedirs(outfname)
|
||||||
|
subf = f.extractfile(info)
|
||||||
|
save_file(outfname, subf)
|
||||||
|
return meta
|
||||||
|
|
||||||
|
def expandhome(s):
|
||||||
|
if "~/" in s:
|
||||||
|
h = os.getenv("HOME")
|
||||||
|
s = s.replace("~/", h + "/")
|
||||||
|
return s
|
||||||
|
|
||||||
|
import ussl
|
||||||
|
import usocket
|
||||||
|
warn_ussl = True
|
||||||
|
def url_open(url):
|
||||||
|
global warn_ussl
|
||||||
|
proto, _, host, urlpath = url.split('/', 3)
|
||||||
|
ai = usocket.getaddrinfo(host, 443)
|
||||||
|
#print("Address infos:", ai)
|
||||||
|
addr = ai[0][4]
|
||||||
|
|
||||||
|
s = usocket.socket(ai[0][0])
|
||||||
|
#print("Connect address:", addr)
|
||||||
|
s.connect(addr)
|
||||||
|
|
||||||
|
if proto == "https:":
|
||||||
|
s = ussl.wrap_socket(s)
|
||||||
|
if warn_ussl:
|
||||||
|
print("Warning: %s SSL certificate is not validated" % host)
|
||||||
|
warn_ussl = False
|
||||||
|
|
||||||
|
# MicroPython rawsocket module supports file interface directly
|
||||||
|
s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host))
|
||||||
|
l = s.readline()
|
||||||
|
protover, status, msg = l.split(None, 2)
|
||||||
|
if status != b"200":
|
||||||
|
if status == b"404":
|
||||||
|
print("Package not found")
|
||||||
|
raise ValueError(status)
|
||||||
|
while 1:
|
||||||
|
l = s.readline()
|
||||||
|
if not l:
|
||||||
|
raise ValueError("Unexpected EOF")
|
||||||
|
if l == b'\r\n':
|
||||||
|
break
|
||||||
|
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def get_pkg_metadata(name):
|
||||||
|
f = url_open("https://pypi.python.org/pypi/%s/json" % name)
|
||||||
|
s = f.read()
|
||||||
|
f.close()
|
||||||
|
return json.loads(s)
|
||||||
|
|
||||||
|
|
||||||
|
def fatal(msg):
|
||||||
|
print(msg)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
def install_pkg(pkg_spec, install_path):
|
||||||
|
data = get_pkg_metadata(pkg_spec)
|
||||||
|
|
||||||
|
latest_ver = data["info"]["version"]
|
||||||
|
packages = data["releases"][latest_ver]
|
||||||
|
del data
|
||||||
|
gc.collect()
|
||||||
|
assert len(packages) == 1
|
||||||
|
package_url = packages[0]["url"]
|
||||||
|
print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url))
|
||||||
|
package_fname = op_basename(package_url)
|
||||||
|
f1 = url_open(package_url)
|
||||||
|
f2 = uzlib.DecompIO(f1, gzdict_sz)
|
||||||
|
f3 = tarfile.TarFile(fileobj=f2)
|
||||||
|
meta = install_tar(f3, install_path)
|
||||||
|
f1.close()
|
||||||
|
del f3
|
||||||
|
del f2
|
||||||
|
gc.collect()
|
||||||
|
return meta
|
||||||
|
|
||||||
|
def install(to_install, install_path=None):
|
||||||
|
# Calculate gzip dictionary size to use
|
||||||
|
global gzdict_sz
|
||||||
|
sz = gc.mem_free() + gc.mem_alloc()
|
||||||
|
if sz <= 65536:
|
||||||
|
gzdict_sz = 16 + 12
|
||||||
|
|
||||||
|
if install_path is None:
|
||||||
|
install_path = get_install_path()
|
||||||
|
if install_path[-1] != "/":
|
||||||
|
install_path += "/"
|
||||||
|
if not isinstance(to_install, list):
|
||||||
|
to_install = [to_install]
|
||||||
|
print("Installing to: " + install_path)
|
||||||
|
# sets would be perfect here, but don't depend on them
|
||||||
|
installed = []
|
||||||
|
try:
|
||||||
|
while to_install:
|
||||||
|
if debug:
|
||||||
|
print("Queue:", to_install)
|
||||||
|
pkg_spec = to_install.pop(0)
|
||||||
|
if pkg_spec in installed:
|
||||||
|
continue
|
||||||
|
meta = install_pkg(pkg_spec, install_path)
|
||||||
|
installed.append(pkg_spec)
|
||||||
|
if debug:
|
||||||
|
print(meta)
|
||||||
|
deps = meta.get("deps", "").rstrip()
|
||||||
|
if deps:
|
||||||
|
deps = deps.decode("utf-8").split("\n")
|
||||||
|
to_install.extend(deps)
|
||||||
|
except NotFoundError:
|
||||||
|
print("Error: cannot find '%s' package (or server error), packages may be partially installed" \
|
||||||
|
% pkg_spec, file=sys.stderr)
|
||||||
|
|
||||||
|
def get_install_path():
|
||||||
|
global install_path
|
||||||
|
if install_path is None:
|
||||||
|
# sys.path[0] is current module's path
|
||||||
|
install_path = sys.path[1]
|
||||||
|
install_path = expandhome(install_path)
|
||||||
|
return install_path
|
||||||
|
|
||||||
|
def cleanup():
|
||||||
|
for fname in cleanup_files:
|
||||||
|
try:
|
||||||
|
os.unlink(fname)
|
||||||
|
except OSError:
|
||||||
|
print("Warning: Cannot delete " + fname)
|
||||||
|
|
||||||
|
def help():
|
||||||
|
print("""\
|
||||||
|
upip - Simple PyPI package manager for MicroPython
|
||||||
|
Usage: micropython -m upip install [-p <path>] <package>... | -r <requirements.txt>
|
||||||
|
import upip; upip.install(package_or_list, [<path>])
|
||||||
|
|
||||||
|
If <path> is not given, packages will be installed into sys.path[1]
|
||||||
|
(can be set from MICROPYPATH environment variable, if current system
|
||||||
|
supports that).""")
|
||||||
|
print("Current value of sys.path[1]:", sys.path[1])
|
||||||
|
print("""\
|
||||||
|
|
||||||
|
Note: only MicroPython packages (usually, named micropython-*) are supported
|
||||||
|
for installation, upip does not support arbitrary code in setup.py.
|
||||||
|
""")
|
||||||
|
|
||||||
|
def main():
|
||||||
|
global debug
|
||||||
|
global install_path
|
||||||
|
install_path = None
|
||||||
|
|
||||||
|
if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
|
||||||
|
help()
|
||||||
|
return
|
||||||
|
|
||||||
|
if sys.argv[1] != "install":
|
||||||
|
fatal("Only 'install' command supported")
|
||||||
|
|
||||||
|
to_install = []
|
||||||
|
|
||||||
|
i = 2
|
||||||
|
while i < len(sys.argv) and sys.argv[i][0] == "-":
|
||||||
|
opt = sys.argv[i]
|
||||||
|
i += 1
|
||||||
|
if opt == "-h" or opt == "--help":
|
||||||
|
help()
|
||||||
|
return
|
||||||
|
elif opt == "-p":
|
||||||
|
install_path = sys.argv[i]
|
||||||
|
i += 1
|
||||||
|
elif opt == "-r":
|
||||||
|
list_file = sys.argv[i]
|
||||||
|
i += 1
|
||||||
|
with open(list_file) as f:
|
||||||
|
while True:
|
||||||
|
l = f.readline()
|
||||||
|
if not l:
|
||||||
|
break
|
||||||
|
to_install.append(l.rstrip())
|
||||||
|
elif opt == "--debug":
|
||||||
|
debug = True
|
||||||
|
else:
|
||||||
|
fatal("Unknown/unsupported option: " + opt)
|
||||||
|
|
||||||
|
to_install.extend(sys.argv[i:])
|
||||||
|
if not to_install:
|
||||||
|
help()
|
||||||
|
return
|
||||||
|
|
||||||
|
install(to_install)
|
||||||
|
|
||||||
|
if not debug:
|
||||||
|
cleanup()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
94
tools/upip_utarfile.py
Normal file
94
tools/upip_utarfile.py
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
import uctypes
|
||||||
|
|
||||||
|
# http://www.gnu.org/software/tar/manual/html_node/Standard.html
|
||||||
|
TAR_HEADER = {
|
||||||
|
"name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100),
|
||||||
|
"size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12),
|
||||||
|
}
|
||||||
|
|
||||||
|
DIRTYPE = "dir"
|
||||||
|
REGTYPE = "file"
|
||||||
|
|
||||||
|
def roundup(val, align):
|
||||||
|
return (val + align - 1) & ~(align - 1)
|
||||||
|
|
||||||
|
class FileSection:
|
||||||
|
|
||||||
|
def __init__(self, f, content_len, aligned_len):
|
||||||
|
self.f = f
|
||||||
|
self.content_len = content_len
|
||||||
|
self.align = aligned_len - content_len
|
||||||
|
|
||||||
|
def read(self, sz=65536):
|
||||||
|
if self.content_len == 0:
|
||||||
|
return b""
|
||||||
|
if sz > self.content_len:
|
||||||
|
sz = self.content_len
|
||||||
|
data = self.f.read(sz)
|
||||||
|
sz = len(data)
|
||||||
|
self.content_len -= sz
|
||||||
|
return data
|
||||||
|
|
||||||
|
def readinto(self, buf):
|
||||||
|
if self.content_len == 0:
|
||||||
|
return 0
|
||||||
|
if len(buf) > self.content_len:
|
||||||
|
buf = memoryview(buf)[:self.content_len]
|
||||||
|
sz = self.f.readinto(buf)
|
||||||
|
self.content_len -= sz
|
||||||
|
return sz
|
||||||
|
|
||||||
|
def skip(self):
|
||||||
|
sz = self.content_len + self.align
|
||||||
|
if sz:
|
||||||
|
buf = bytearray(16)
|
||||||
|
while sz:
|
||||||
|
s = min(sz, 16)
|
||||||
|
self.f.readinto(buf, s)
|
||||||
|
sz -= s
|
||||||
|
|
||||||
|
class TarInfo:
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size)
|
||||||
|
|
||||||
|
class TarFile:
|
||||||
|
|
||||||
|
def __init__(self, name=None, fileobj=None):
|
||||||
|
if fileobj:
|
||||||
|
self.f = fileobj
|
||||||
|
else:
|
||||||
|
self.f = open(name, "rb")
|
||||||
|
self.subf = None
|
||||||
|
|
||||||
|
def next(self):
|
||||||
|
if self.subf:
|
||||||
|
self.subf.skip()
|
||||||
|
buf = self.f.read(512)
|
||||||
|
if not buf:
|
||||||
|
return None
|
||||||
|
|
||||||
|
h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN)
|
||||||
|
|
||||||
|
# Empty block means end of archive
|
||||||
|
if h.name[0] == 0:
|
||||||
|
return None
|
||||||
|
|
||||||
|
d = TarInfo()
|
||||||
|
d.name = str(h.name, "utf-8").rstrip()
|
||||||
|
d.size = int(bytes(h.size).rstrip(), 8)
|
||||||
|
d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"]
|
||||||
|
self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512))
|
||||||
|
return d
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __next__(self):
|
||||||
|
v = self.next()
|
||||||
|
if v is None:
|
||||||
|
raise StopIteration
|
||||||
|
return v
|
||||||
|
|
||||||
|
def extractfile(self, tarinfo):
|
||||||
|
return tarinfo.subf
|
@ -1,6 +1,8 @@
|
|||||||
-include mpconfigport.mk
|
-include mpconfigport.mk
|
||||||
include ../py/mkenv.mk
|
include ../py/mkenv.mk
|
||||||
|
|
||||||
|
FROZEN_DIR = scripts
|
||||||
|
|
||||||
# define main target
|
# define main target
|
||||||
PROG = micropython
|
PROG = micropython
|
||||||
|
|
||||||
@ -148,17 +150,6 @@ SRC_C = \
|
|||||||
fatfs_port.c \
|
fatfs_port.c \
|
||||||
$(SRC_MOD)
|
$(SRC_MOD)
|
||||||
|
|
||||||
# Include builtin package manager in the standard build (and coverage)
|
|
||||||
ifeq ($(PROG),micropython)
|
|
||||||
SRC_C += $(BUILD)/_frozen_upip.c
|
|
||||||
else ifeq ($(PROG),micropython_coverage)
|
|
||||||
SRC_C += $(BUILD)/_frozen_upip.c
|
|
||||||
else ifeq ($(PROG), micropython_nanbox)
|
|
||||||
SRC_C += $(BUILD)/_frozen_upip.c
|
|
||||||
else ifeq ($(PROG), micropython_freedos)
|
|
||||||
SRC_C += $(BUILD)/_frozen_upip.c
|
|
||||||
endif
|
|
||||||
|
|
||||||
LIB_SRC_C = $(addprefix lib/,\
|
LIB_SRC_C = $(addprefix lib/,\
|
||||||
$(LIB_SRC_C_EXTRA) \
|
$(LIB_SRC_C_EXTRA) \
|
||||||
timeutils/timeutils.c \
|
timeutils/timeutils.c \
|
||||||
@ -235,7 +226,7 @@ fast:
|
|||||||
# build a minimal interpreter
|
# build a minimal interpreter
|
||||||
minimal:
|
minimal:
|
||||||
$(MAKE) COPT="-Os -DNDEBUG" CFLAGS_EXTRA='-DMP_CONFIGFILE="<mpconfigport_minimal.h>"' \
|
$(MAKE) COPT="-Os -DNDEBUG" CFLAGS_EXTRA='-DMP_CONFIGFILE="<mpconfigport_minimal.h>"' \
|
||||||
BUILD=build-minimal PROG=micropython_minimal \
|
BUILD=build-minimal PROG=micropython_minimal FROZEN_DIR= \
|
||||||
MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_SOCKET=0 MICROPY_PY_THREAD=0 \
|
MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_SOCKET=0 MICROPY_PY_THREAD=0 \
|
||||||
MICROPY_PY_TERMIOS=0 MICROPY_PY_USSL=0 \
|
MICROPY_PY_TERMIOS=0 MICROPY_PY_USSL=0 \
|
||||||
MICROPY_USE_READLINE=0 MICROPY_FATFS=0
|
MICROPY_USE_READLINE=0 MICROPY_FATFS=0
|
||||||
@ -272,22 +263,6 @@ coverage_test: coverage
|
|||||||
gcov -o build-coverage/py ../py/*.c
|
gcov -o build-coverage/py ../py/*.c
|
||||||
gcov -o build-coverage/extmod ../extmod/*.c
|
gcov -o build-coverage/extmod ../extmod/*.c
|
||||||
|
|
||||||
$(BUILD)/_frozen_upip.c: $(BUILD)/frozen_upip/upip.py
|
|
||||||
$(MAKE_FROZEN) $(dir $^) > $@
|
|
||||||
|
|
||||||
# Select latest upip version available
|
|
||||||
UPIP_TARBALL := $(shell ls -1 -v ../tools/micropython-upip-*.tar.gz | tail -n1)
|
|
||||||
|
|
||||||
$(BUILD)/frozen_upip/upip.py: $(UPIP_TARBALL)
|
|
||||||
$(ECHO) "MISC Preparing upip as frozen module"
|
|
||||||
$(Q)mkdir -p $(BUILD)
|
|
||||||
$(Q)rm -rf $(BUILD)/micropython-upip-*
|
|
||||||
$(Q)tar -C $(BUILD) -xz -f $^
|
|
||||||
$(Q)rm -rf $(dir $@)
|
|
||||||
$(Q)mkdir -p $(dir $@)
|
|
||||||
$(Q)cp $(BUILD)/micropython-upip-*/upip*.py $(dir $@)
|
|
||||||
|
|
||||||
|
|
||||||
# Value of configure's --host= option (required for cross-compilation).
|
# Value of configure's --host= option (required for cross-compilation).
|
||||||
# Deduce it from CROSS_COMPILE by default, but can be overriden.
|
# Deduce it from CROSS_COMPILE by default, but can be overriden.
|
||||||
ifneq ($(CROSS_COMPILE),)
|
ifneq ($(CROSS_COMPILE),)
|
||||||
|
Loading…
Reference in New Issue
Block a user