Commit ff4ca38d authored by Jan Frederick Eick's avatar Jan Frederick Eick
Browse files

baz

parent 42f24a8b
......@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 247,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
......@@ -14,7 +14,7 @@
},
{
"cell_type": "code",
"execution_count": 203,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
......@@ -27,7 +27,7 @@
},
{
"cell_type": "code",
"execution_count": 204,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
......@@ -40,7 +40,7 @@
},
{
"cell_type": "code",
"execution_count": 205,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
......@@ -49,7 +49,7 @@
},
{
"cell_type": "code",
"execution_count": 206,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
......@@ -58,7 +58,7 @@
},
{
"cell_type": "code",
"execution_count": 207,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
......@@ -68,7 +68,7 @@
},
{
"cell_type": "code",
"execution_count": 208,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
......@@ -78,7 +78,7 @@
},
{
"cell_type": "code",
"execution_count": 209,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
......@@ -87,7 +87,7 @@
},
{
"cell_type": "code",
"execution_count": 210,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
......@@ -112,7 +112,7 @@
},
{
"cell_type": "code",
"execution_count": 211,
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
......@@ -126,7 +126,7 @@
},
{
"cell_type": "code",
"execution_count": 212,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
......@@ -138,7 +138,7 @@
},
{
"cell_type": "code",
"execution_count": 213,
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
......@@ -150,7 +150,7 @@
},
{
"cell_type": "code",
"execution_count": 214,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
......@@ -172,7 +172,7 @@
},
{
"cell_type": "code",
"execution_count": 215,
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
......@@ -181,7 +181,37 @@
},
{
"cell_type": "code",
"execution_count": 216,
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 1.29903811, 0. , 0. , 0. ],\n",
" [ 0. , 1.73205081, 0. , 0. ],\n",
" [ 0. , 0. , -1.002002 , -2.002002 ],\n",
" [ 0. , 0. , -1. , 0. ]])"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"projMat"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
......@@ -190,31 +220,64 @@
},
{
"cell_type": "code",
"execution_count": 217,
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"MInv = np.linalg.inv(M)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"def applyMat4(v, m):\n",
" w = 1 / (m[0,3] * v[0] + m[1,3] * v[1] + m[2,3] * v[2] + m[3,3])\n",
" x = (m[0,0] * v[0] + m[1,0] * v[1] + m[2,0] * v[2] + m[3,0]) * w\n",
" y = (m[0,1] * v[0] + m[1,1] * v[1] + m[2,1] * v[2] + m[3,1]) * w\n",
" z = (m[0,2] * v[0] + m[1,2] * v[1] + m[2,2] * v[2] + m[3,2]) * w\n",
" #w = 1 / (m[0,3] * v[0] + m[1,3] * v[1] + m[2,3] * v[2] + m[3,3])\n",
" #x = (m[0,0] * v[0] + m[1,0] * v[1] + m[2,0] * v[2] + m[3,0]) * w\n",
" #y = (m[0,1] * v[0] + m[1,1] * v[1] + m[2,1] * v[2] + m[3,1]) * w\n",
" #z = (m[0,2] * v[0] + m[1,2] * v[1] + m[2,2] * v[2] + m[3,2]) * w\n",
" w = 1 / (m[3,0] * v[0] + m[3,1] * v[1] + m[3,2] * v[2] + m[3,3])\n",
" x = (m[0,0] * v[0] + m[0,1] * v[1] + m[0,2] * v[2] + m[0,3]) * w\n",
" y = (m[1,0] * v[0] + m[1,1] * v[1] + m[2,1] * v[2] + m[1,3]) * w\n",
" z = (m[2,0] * v[0] + m[2,1] * v[1] + m[2,2] * v[2] + m[2,3]) * w\n",
" return np.array([x, y, z])"
]
},
{
"cell_type": "code",
"execution_count": 218,
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"tmp = applyMat4(coord_vector, projMatInv.T)\n",
"unproject = applyMat4(tmp, M.T)"
"tmp = applyMat4(coord_vector, projMatInv)\n",
"unproject = applyMat4(tmp, M)"
]
},
{
"cell_type": "code",
"execution_count": 219,
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 3.98803589, -0.09593723, 0.64757632])"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"unproject"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
......@@ -223,7 +286,27 @@
},
{
"cell_type": "code",
"execution_count": 220,
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([98.67933615, -2.37385589, 16.02352724])"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"unproject * 100"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
......@@ -232,7 +315,7 @@
},
{
"cell_type": "code",
"execution_count": 314,
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
......@@ -241,8 +324,8 @@
" coord_y = -(y/h) * 2 + 1\n",
" coord_z = 0.5\n",
" coord_vector = np.array([coord_x, coord_y, coord_z, 1])\n",
" tmp = applyMat4(coord_vector, projMatInv.T)\n",
" ray_direction = applyMat4(tmp, matWorld.T)\n",
" tmp = applyMat4(coord_vector, projMatInv)\n",
" ray_direction = applyMat4(tmp, matWorld)\n",
" ray_direction /= np.linalg.norm(ray_direction)\n",
" d = -ray_direction\n",
" u = 0.5 + math.atan2(d[2], d[0]) / (2 * math.pi)\n",
......@@ -254,7 +337,7 @@
},
{
"cell_type": "code",
"execution_count": 225,
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
......@@ -267,7 +350,7 @@
},
{
"cell_type": "code",
"execution_count": 226,
"execution_count": 26,
"metadata": {},
"outputs": [
{
......@@ -276,7 +359,7 @@
"(209, 2078)"
]
},
"execution_count": 226,
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
......@@ -287,7 +370,7 @@
},
{
"cell_type": "code",
"execution_count": 249,
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
......@@ -296,7 +379,7 @@
},
{
"cell_type": "code",
"execution_count": 318,
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
......@@ -311,48 +394,38 @@
},
{
"cell_type": "code",
"execution_count": 319,
"execution_count": 29,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'cartesianX': array([0. , 0. , 0. , ..., 0.99156222, 0.98841063,\n",
" 0.98629433]),\n",
" 'cartesianY': array([-0. , -0. , -0. , ..., 0.01216891,\n",
" 0.00606489, 0. ]),\n",
" 'cartesianZ': array([ 1.02932966, 1.02705956, 1.02493954, ..., -1.70145392,\n",
" -1.69595027, -1.69228709]),\n",
" 'cartesianInvalidState': array([0, 0, 0, ..., 0, 0, 0], dtype=int8),\n",
" 'colorRed': array([255, 255, 255, ..., 88, 88, 88], dtype=uint8),\n",
" 'colorGreen': array([255, 255, 255, ..., 88, 88, 88], dtype=uint8),\n",
" 'colorBlue': array([255, 255, 255, ..., 88, 88, 88], dtype=uint8),\n",
" 'intensity': array([0., 0., 0., ..., 0., 0., 0.], dtype=float32),\n",
" 'rowIndex': array([511, 511, 511, ..., 85, 85, 85], dtype=uint16),\n",
" 'columnIndex': array([ 210, 220, 233, ..., 1021, 1022, 1023], dtype=uint16)}"
"1023"
]
},
"execution_count": 319,
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": []
"source": [
"colMax"
]
},
{
"cell_type": "code",
"execution_count": 230,
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"pan_im = Image.open('./panoramas/2057-5bc353cc-846d-42c9-ae92-fb44278f3873-gray-8192.jpg')\n",
"pan_im2 = pan_im.copy()\n",
"#pan_im = Image.open('./panoramas/2057-5bc353cc-846d-42c9-ae92-fb44278f3873-gray-8192.jpg')\n",
"#pan_im2 = pan_im.copy()\n",
"im = Image.open('./segmented/2057-5bc353cc-846d-42c9-ae92-fb44278f3873-gray-8192_0_0.png')"
]
},
{
"cell_type": "code",
"execution_count": 231,
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
......@@ -361,47 +434,155 @@
},
{
"cell_type": "code",
"execution_count": 320,
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"for x in range(im.width):\n",
" #print(x)\n",
" for y in range(im.height):\n",
" if px[x, y] == (255, 0, 0, 255):\n",
" pano_coords = calcPanoCoord(x, y, 1024, 768, projMatInv, M)\n",
" pan_im2.putpixel(pano_coords, (255, 0, 0))"
"#for x in range(im.width):\n",
"# #print(x)\n",
"# for y in range(im.height):\n",
"# if px[x, y] == (255, 0, 0, 255):\n",
"# pano_coords = calcPanoCoord(x, y, 1024, 768, projMatInv, M)\n",
"# pan_im2.putpixel(pano_coords, (255, 0, 0))"
]
},
{
"cell_type": "code",
"execution_count": 241,
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
"#pan_im2.getpixel((8168,2379))"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"#pan_im2.save(\"./panorama_segmented/test.png\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(255, 0, 0)"
"(620, 400)"
]
},
"execution_count": 241,
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pan_im2.getpixel((8168,2379))"
"def panoToScreen(pano_x, pano_y, pano_w, pano_h, MInv, projMat):\n",
" u = (pano_x - 0.5) / pano_w\n",
" v = 1 - (pano_y - 0.5) / pano_h\n",
" radius = 100\n",
" u = 2 * math.pi * u\n",
" v = math.pi * v\n",
"\n",
" x = math.cos(u) * math.sin(v) * radius\n",
" z = math.sin(v) * math.sin(u) * radius\n",
" #x = -math.sin(u) * math.sin(v)\n",
" #y = math.cos(u) * math.sin(v)\n",
" y = -math.cos(v) * radius\n",
" p = np.array([x, y, z])\n",
" tmp = applyMat4(p, MInv)\n",
" #tmp = np.array([p[0], p[2], p[1]])\n",
" #tmp = applyMat4(tmp, MInv)\n",
" p2 = applyMat4(tmp, projMat)\n",
" #p2 = applyMat4(p, projMat)\n",
" #p2 /= p2[2] / 0.5\n",
" cam_x = round(1024 / 2 + p2[0] * 1024/2 + 1)\n",
" cam_y = round(768 /2 - p2[1] * 768 / 2 + 1)\n",
" return (cam_x, cam_y)\n",
"\n",
"panoToScreen(pano_x, pano_y, pano_w, pano_h, MInv, projMat)"
]
},
{
"cell_type": "code",
"execution_count": 317,
"execution_count": 41,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"finished\n"
]
}
],
"source": [
"pan_im2.save(\"./panorama_segmented/test.png\")"
"#e57_write = pye57.E57(\"test.e57\", mode=\"w\")\n",
"with open(\"test.xyz\", \"w\") as xyzfile:\n",
" for i in range(1, header.point_count):\n",
" pano_y = data_raw[\"rowIndex\"][i]\n",
" pano_x = data_raw[\"columnIndex\"][i]\n",
" x, y, z = data_raw['cartesianX'][i], data_raw['cartesianY'][i], data_raw['cartesianZ'][i]\n",
" r, g, b = data_raw['colorRed'][i], data_raw['colorGreen'][i], data_raw['colorBlue'][i]\n",
" screen_coords = panoToScreen(pano_x, pano_y, colMax, rowMax, MInv, projMat)\n",
" #if screen_coords[0] < 0 or screen_coords[0] > 1023 or screen_coords[1] < 0 or screen_coords[1] > 767:\n",
" # continue\n",
" \n",
" #col = im.getpixel(screen_coords)\n",
" if col == (255, 0, 0, 255):\n",
" print('writing color in laserscan')\n",
" #data_raw['colorRed'][i] = 255\n",
" #data_raw['colorGreen'][i] = 0\n",
" #data_raw['colorBlue'][i] = 0\n",
" r = 255\n",
" g = 0\n",
" b = 0\n",
" \n",
" xyzfile.write(f\"{x} {y} {z} {r} {g} {b}\\n\")\n",
"\n",
"print(\"finished\")"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"397207"
]
},
"execution_count": 39,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"header.point_count"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
......
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from pythreejs import *\n",
"from IPython.display import display\n",
"from ipywebrtc import WidgetStream, ImageRecorder\n",
"import ipywidgets as widgets\n",
"from math import pi\n",
"import math\n",
"import PIL.Image\n",
"import io\n",
"import time\n",
"import os\n",
"import glob\n",
"from IPython.display import Javascript\n",
"import base64"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# Reduce repo churn for examples with embedded state:\n",
"from pythreejs._example_helper import use_example_model_ids\n",
"use_example_model_ids()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"view_width = 1024\n",
"view_height = 768"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"panoramas = glob.glob(\"2020-11-11-BA1-indoor-pano/*.jpg\")\n",
"panoramas.sort()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"pano_tex = ImageTexture(imageUri=panoramas[1])\n",
"pano_material = MeshBasicMaterial(map=pano_tex, side='BackSide')\n",
"sphere = SphereGeometry(100, 100, 40)\n",
"sphere_mesh = Mesh(\n",
" sphere,\n",
" pano_material\n",
")\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"camera = PerspectiveCamera(fov=60, position=[0, 0, 0], aspect=view_width/view_height)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"sphere_mesh.scale = (-1,1,1)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"scene = Scene(children=[sphere_mesh, camera, AmbientLight(color='#777777')])\n",
"#controller = OrbitControls(controlling=camera)\n",
"renderer = Renderer(camera=camera, scene=scene, \n",
" width=view_width, height=view_height)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "pythree_example_model_008",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Renderer(camera=PerspectiveCamera(aspect=1.3333333333333333, fov=60.0, position=(0.0, 0.0, 0.0), projectionMat…"
]
},
"metadata": {},
"output_type": "display_data"