Merge branch 'main' of https://github.com/charlesbvll/monoloco into main

* Only dark theme with webcam

* Fixed axes colors

* Update printer.py

* Fixed axes in printer.py

* Fixed test images names

* Fixed README

* Fixed README

* Fixed bird view

* Linting

* Linting

.gitattributes

@@ -0,0 +1 @@
+monoloco/_version.py export-subst

.github/workflows/tests.yml

@@ -0,0 +1,96 @@
+
+# Adapted from https://github.com/openpifpaf/openpifpaf/blob/main/.github/workflows/test.yml,
+#which is: 'Copyright 2019-2021 by Sven Kreiss and contributors. All rights reserved.'
+# and licensed under GNU AGPLv3
+
+name: Tests
+
+on:
+  push:
+    paths:
+      - 'monoloco/**'
+      - 'test/**'
+      - 'docs/00*.png'
+      - 'docs/frame0032.jpg'
+      - '.github/workflows/tests.yml'
+
+  pull_request:
+    paths:
+      - 'monoloco/**'
+      - 'test/**'
+      - 'docs/00*.png'
+      - 'docs/frame0032.jpg'
+      - '.github/workflows/tests.yml'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            python: 3.7
+            torch: 1.7.1+cpu
+            torchvision: 0.8.2+cpu
+            torch-source: https://download.pytorch.org/whl/torch_stable.html
+          - os: ubuntu-latest
+            python: 3.8
+            torch: 1.7.1+cpu
+            torchvision: 0.8.2+cpu
+            torch-source: https://download.pytorch.org/whl/cpu/torch_stable.html
+          - os: macos-latest
+            python: 3.7
+            torch: 1.7.1
+            torchvision: 0.8.2
+            torch-source: https://download.pytorch.org/whl/torch_stable.html
+          - os: macos-latest
+            python: 3.8
+            torch: 1.7.1
+            torchvision: 0.8.2
+            torch-source: https://download.pytorch.org/whl/torch_stable.html
+          - os: windows-latest
+            python: 3.7
+            torch: 1.7.1+cpu
+            torchvision: 0.8.2+cpu
+            torch-source: https://download.pytorch.org/whl/torch_stable.html
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python }}
+      if: ${{ !matrix.conda }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Set up Conda
+      if: matrix.conda
+      uses: s-weigand/setup-conda@v1
+      with:
+        update-conda: true
+        python-version: ${{ matrix.python }}
+        conda-channels: anaconda, conda-forge
+    - run: conda --version
+      if: matrix.conda
+    - run: which python
+      if: matrix.conda
+    - run: python --version
+    - name: Install
+      run: |
+        python -m pip install --upgrade pip setuptools
+        python -m pip install -e ".[test]"
+    - name: Print environment
+      run: |
+        python -m pip freeze
+        python --version
+        python -c "import monoloco; print(monoloco.__version__)"
+    - name: Lint monoloco
+      run: |
+        pylint monoloco --disable=fixme
+    - name: Lint tests
+      if: matrix.os != 'windows-latest'  # because of path separator
+      run: |
+        pylint tests/*.py --disable=fixme
+
+    - name: Test
+      run: |
+        pytest -vv

.gitignore

@@ -1,11 +1,14 @@
 .idea/
-data/
+data
 .DS_store
 __pycache__
-Monoloco/*.pyc
+monoloco/*.pyc
 .pytest*
-dist/
 build/
+dist/
 *.egg-info
 tests/*.png
-kitti-eval/*
+kitti-eval/build
+kitti-eval/cmake-build-debug
+figures/
+visual_tests/

.pylintrc

@@ -1,26 +0,0 @@
-
-
-[BASIC]
-variable-rgx=[a-z0-9_]{1,30}$  # to accept 2 (dfferent) letters variables
-
-
-Good-names=xx,dd,zz,hh,ww,pp,kk,lr,w1,w2,w3,mm,im,uv,ax,COV_MIN,CONF_MIN
-
-
-[TYPECHECK]
-
-disable=E1102,missing-docstring,useless-object-inheritance,duplicate-code,too-many-arguments,too-many-instance-attributes,too-many-locals,too-few-public-methods,arguments-differ,logging-format-interpolation
-
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-
-generated-members=numpy.*,torch.*,cv2.*
-
-ignored-modules=nuscenes, tabulate, cv2
-
-
-
-[FORMAT]
-max-line-length=120

.travis.yml

@@ -1,11 +0,0 @@
-dist: xenial
-language: python
-python:
- - "3.6"
- - "3.7"
-install:
- - pip install --upgrade pip setuptools
- - pip install ".[test]"
-script:
- - pylint monoloco --disable=unused-variable,fixme
- - pytest -v

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2019 by EPFL/VITA. All rights reserved.
+Copyright 2018-2021 by Lorenzo Bertoni and contributors. All rights reserved.
 
 This project and all its files are licensed under
 GNU AGPLv3 or later version.
@@ -6,4 +6,15 @@ GNU AGPLv3 or later version.
 If this license is not suitable for your business or project
 please contact EPFL-TTO (https://tto.epfl.ch/) for a full commercial license.
 
-This software may not be used to harm any person deliberately.
+This software may not be used to harm any person deliberately or for any military application.
+
+
+The following files are based on the OpenPifPaf project which is
+"Copyright 2019-2021 by Sven Kreiss and contributors. All rights reserved." and licensed under GNU AGPLv3.
+
+- tests/test_train_mono.py
+- tests/test_train_stereo.py
+- monoloco/visuals/pifpaf_show.py
+- monoloco/train/losses.py
+- monoloco/predict.py
+-.github/workflows/tests.yml

LICENSE.AGPL

@@ -1,661 +0,0 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<http://www.gnu.org/licenses/>.

MANIFEST.in

@@ -0,0 +1,2 @@
+include versioneer.py
+include monoloco/_version.py

README.md

@@ -1,83 +1,78 @@
-# Monoloco
+# Monoloco library      [![Downloads](https://pepy.tech/badge/monoloco)](https://pepy.tech/project/monoloco)
+Continuously tested on Linux, MacOS and Windows: [![Tests](https://github.com/vita-epfl/monoloco/workflows/Tests/badge.svg)](https://github.com/vita-epfl/monoloco/actions?query=workflow%3ATests)
 
-> We tackle the fundamentally ill-posed problem of 3D human localization from monocular RGB images. Driven by the limitation of neural networks outputting point estimates, we address the ambiguity in the task by predicting confidence intervals through a loss function based on the Laplace distribution. Our architecture is a light-weight feed-forward neural network that predicts 3D locations and corresponding confidence intervals given 2D human poses. The design is particularly well suited for small training data, cross-dataset generalization, and real-time applications. Our experiments show that we (i) outperform state-of-the-art results on KITTI and nuScenes datasets, (ii) even outperform a stereo-based method for far-away pedestrians, and (iii) estimate meaningful confidence intervals. We further share insights on our model of uncertainty in cases of limited observations and out-of-distribution samples.
 
-```
+<img src="docs/webcam.gif" width="700" alt="gif" />
-@InProceedings{Bertoni_2019_ICCV,
-author = {Bertoni, Lorenzo and Kreiss, Sven and Alahi, Alexandre},
-title = {MonoLoco: Monocular 3D Pedestrian Localization and Uncertainty Estimation},
-booktitle = {The IEEE International Conference on Computer Vision (ICCV)},
-month = {October},
-year = {2019}
-}
-```
-**2021**
 
-**NEW! MonoLoco++ is [available](https://github.com/vita-epfl/monstereo):**
+<br /> 
-* It estimates 3D localization, orientation, and bounding box dimensions
+<br /> 
-* It verifies social distance requirements. More info: [video](https://www.youtube.com/watch?v=r32UxHFAJ2M) and [project page](http://vita.epfl.ch/monoloco)
-* It works with [OpenPifPaf](https://github.com/vita-epfl/openpifpaf) 0.12 and PyTorch 1.7
 
-**2020** 
+This library is based on three research projects for monocular/stereo 3D human localization (detection), body orientation, and social distancing. Check the __video teaser__ of the library on [__YouTube__](https://www.youtube.com/watch?v=O5zhzi8mwJ4). 
-* Paper on [ICCV'19](http://openaccess.thecvf.com/content_ICCV_2019/html/Bertoni_MonoLoco_Monocular_3D_Pedestrian_Localization_and_Uncertainty_Estimation_ICCV_2019_paper.html) website or [ArXiv](https://arxiv.org/abs/1906.06059)
-* Check our video with method description and qualitative results on [YouTube](https://www.youtube.com/watch?v=ii0fqerQrec)
 
-* Live demo available! (more info in the webcam section)
 
-* Continuously tested with Travis CI: [![Build Status](https://travis-ci.org/vita-epfl/monoloco.svg?branch=master)](https://travis-ci.org/vita-epfl/monoloco)<br />
+---
 
-<img src="docs/pull.png" height="600">
+> __MonStereo: When Monocular and Stereo Meet at the Tail of 3D Human Localization__<br /> 
+> _[L. Bertoni](https://scholar.google.com/citations?user=f-4YHeMAAAAJ&hl=en), [S. Kreiss](https://www.svenkreiss.com), 
+[T. Mordan](https://people.epfl.ch/taylor.mordan/?lang=en), [A. Alahi](https://scholar.google.com/citations?user=UIhXQ64AAAAJ&hl=en)_, ICRA 2021 <br /> 
+__[Article](https://arxiv.org/abs/2008.10913)__  &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;   __[Citation](#Citation)__  &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; __[Video](https://www.youtube.com/watch?v=pGssROjckHU)__
      
-# Setup
+<img src="docs/out_000840_multi.jpg" width="700"/>
 
-### Install
+---
-Python 3 is required. Python 2 is not supported. 
+
-Do not clone this repository and make sure there is no folder named monoloco in your current directory.
+
+> __Perceiving Humans: from Monocular 3D Localization to Social Distancing__<br />
+> _[L. Bertoni](https://scholar.google.com/citations?user=f-4YHeMAAAAJ&hl=en), [S. Kreiss](https://www.svenkreiss.com), 
+[A. Alahi](https://scholar.google.com/citations?user=UIhXQ64AAAAJ&hl=en)_, T-ITS 2021 <br /> 
+__[Article](https://arxiv.org/abs/2009.00984)__ &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; __[Citation](#Citation)__ &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;  &nbsp; &nbsp; &nbsp; __[Video](https://www.youtube.com/watch?v=r32UxHFAJ2M)__ 
+
+<img src="docs/social_distancing.jpg" width="700"/>
+
+---
+
+> __MonoLoco: Monocular 3D Pedestrian Localization and Uncertainty Estimation__<br />
+> _[L. Bertoni](https://scholar.google.com/citations?user=f-4YHeMAAAAJ&hl=en), [S. Kreiss](https://www.svenkreiss.com), [A.Alahi](https://scholar.google.com/citations?user=UIhXQ64AAAAJ&hl=en)_, ICCV 2019 <br /> 
+__[Article](https://arxiv.org/abs/1906.06059)__ &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;    __[Citation](#Citation)__ &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;  __[Video](https://www.youtube.com/watch?v=ii0fqerQrec)__
+   
+<img src="docs/surf.jpg" width="700"/>
+
+## Library Overview
+Visual illustration of the library components:
+
+<img src="docs/monoloco.gif" width="700" alt="gif" />
+
+## License
+All projects are built upon [Openpifpaf](https://github.com/vita-epfl/openpifpaf) for the 2D keypoints and share the AGPL Licence.
+
+This software is also available for commercial licensing via the EPFL Technology Transfer
+Office (https://tto.epfl.ch/, info.tto@epfl.ch).
+
+
+## Quick setup
+A GPU is not required, yet highly recommended for real-time performances. 
+
+The installation has been tested on OSX and Linux operating systems, with Python 3.6, 3.7, 3.8. 
+Packages have been installed with pip and virtual environments.
+
+For quick installation, do not clone this repository, make sure there is no folder named monoloco in your current directory, and run:
 
 ```
 pip3 install monoloco
+pip3 install matplotlib
 ```
 
-For development of the monoloco source code itself, you need to clone this repository and then:
+For development of the source code itself, you need to clone this repository and then:
 ```
-pip3 install -e '.[test, prep]'
+pip3 install sdist
-```
+cd monoloco
-Python 3.6 or 3.7 is required for nuScenes development kit. 
+python3 setup.py sdist bdist_wheel
-All details for Pifpaf pose detector at [openpifpaf](https://github.com/vita-epfl/openpifpaf).
+pip3 install -e .
-
-
-
-### Data structure
-
-    Data         
-    ├── arrays                 
-    ├── models
-    ├── kitti
-    ├── nuscenes
-    ├── logs
-    
-
-Run the following to create the folders:
-```
-mkdir data
-cd data
-mkdir arrays models kitti nuscenes logs
 ```
 
-### Pre-trained Models
+### Interfaces
-* Download a MonoLoco pre-trained model from 
+All the commands are run through a main file called `run.py` using subparsers.
-[Google Drive](https://drive.google.com/open?id=1F7UG1HPXGlDD_qL-AN5cv2Eg-mhdQkwv) and save it in `data/models` 
+To check all the options:
-(default) or in any folder and call it through the command line option `--model <model path>`
-* Pifpaf pre-trained model will be automatically downloaded at the first run. 
-Three standard, pretrained models are available when using the command line option 
-`--checkpoint resnet50`, `--checkpoint resnet101` and `--checkpoint resnet152`.
-Alternatively, you can download a Pifpaf pre-trained model from [openpifpaf](https://github.com/vita-epfl/openpifpaf)
- and call it with `--checkpoint  <pifpaf model path>`
-
-
-# Interfaces
-All the commands are run through a main file called `main.py` using subparsers.
-To check all the commands for the parser and the subparsers (including openpifpaf ones) run:
 
 * `python3 -m monoloco.run --help`
 * `python3 -m monoloco.run predict --help`
@@ -87,117 +82,334 @@ To check all the commands for the parser and the subparsers (including openpifpa
 
 or check the file `monoloco/run.py`
 
-# Prediction
+#  Predictions
-The predict script receives an image (or an entire folder using glob expressions), 
+
-calls PifPaf for 2d human pose detection over the image
+The software receives an image (or an entire folder using glob expressions), 
-and runs Monoloco for 3d location of the detected poses.
+calls PifPaf for 2D human pose detection over the image
-The command `--networks` defines if saving pifpaf outputs, MonoLoco outputs or both.
+and runs Monoloco++ or MonStereo for 3D localization &/or social distancing &/or orientation
-You can check all commands for Pifpaf at [openpifpaf](https://github.com/vita-epfl/openpifpaf).
+
+**Which Modality** <br /> 
+The command `--mode` defines which network to run.
+
+- select `--mode mono` (default) to predict the 3D localization of all the humans from monocular image(s)
+- select `--mode stereo` for stereo images
+- select `--mode keypoints` if just interested in 2D keypoints from OpenPifPaf
+
+Models are downloaded automatically. To use a specific model, use the command `--model`. Additional models can be downloaded from [here](https://drive.google.com/drive/folders/1jZToVMBEZQMdLB5BAIq2CdCLP5kzNo9t?usp=sharing)
+
+**Which Visualization** <br /> 
+- select `--output_types multi` if you want to visualize both frontal view or bird's eye view in the same picture
+- select `--output_types bird front` if you want to different pictures for the two views or just one view
+- select `--output_types json` if you'd like the ouput json file
+
+If you select `--mode keypoints`, use standard OpenPifPaf arguments
+
+**Focal Length and Camera Parameters** <br /> 
+Absolute distances are affected by the camera intrinsic parameters. 
+When processing KITTI images, the network uses the provided intrinsic matrix of the dataset. 
+In all the other cases, we use the parameters of nuScenes cameras, with "1/1.8'' CMOS sensors of size 7.2 x 5.4 mm.
+The default focal length is 5.7mm and this parameter can be modified using the argument `--focal`.
 
 
-Output options include json files and/or visualization of the predictions on the image in *frontal mode*, 
+## A) 3D Localization
-*birds-eye-view mode* or *combined mode* and can be specified with `--output_types`
 
-
+**Ground-truth comparison** <br /> 
-### Ground truth matching
+If you provide a ground-truth json file to compare the predictions of the network,
-* In case you provide a ground-truth json file to compare the predictions of MonoLoco,
  the script will match every detection using Intersection over Union metric. 
- The ground truth file can be generated using the subparser `prep` and called with the command `--path_gt`.
+ The ground truth file can be generated using the subparser `prep`, or directly downloaded from [Google Drive](https://drive.google.com/file/d/1e-wXTO460ip_Je2NdXojxrOrJ-Oirlgh/view?usp=sharing) 
- Check preprocess section for more details or download the file from 
+ and called it with the command `--path_gt`. 
- [here](https://drive.google.com/open?id=1F7UG1HPXGlDD_qL-AN5cv2Eg-mhdQkwv).
- 
-* In case you don't provide a ground-truth file, the script will look for a predefined path. 
-If it does not find the file, it will generate images
-with all the predictions without ground-truth matching.
-
-Below an example with and without ground-truth matching. They have been created (adding or removing `--path_gt`) with:
-`python3 -m monoloco.run predict --glob docs/002282.png --output_types combined --scale 2 
---model data/models/monoloco-190513-1437.pkl --n_dropout 50 --z_max 30`
- 
-With ground truth matching (only matching people):
-![predict_ground_truth](docs/002282.png.combined_1.png)
-
-Without ground_truth matching (all the detected people): 
-![predict_no_matching](docs/002282.png.combined_2.png)
-
-### Images without calibration matrix
-To accurately estimate distance, the focal length is necessary. 
-However, it is still possible to test Monoloco on images where the calibration matrix is not available. 
-Absolute distances are not meaningful but relative distance still are. 
-Below an example on a generic image from the web, created with:
-`python3 -m monoloco.run predict --glob docs/surf.jpg --output_types combined --model data/models/monoloco-190513-1437.pkl --n_dropout 50 --z_max 25`
-
-![no calibration](docs/surf.jpg.combined.png)
 
 
-# Webcam
+**Monocular examples** <br>
-<img src="docs/webcam_short.gif" height=350 alt="example image" />
 
-MonoLoco can run on personal computers with only CPU and low resolution images (e.g. 256x144) at ~2fps.
+For an example image, run the following command:
-It support 3 types of visualizations: `front`, `bird` and `combined`.
-Multiple visualizations can be combined in different windows.
 
-The above gif has been obtained running on a Macbook the command:
+```sh
-
+python3 -m monoloco.run predict docs/002282.png \
-```pip3 install opencv-python
+--path_gt names-kitti-200615-1022.json \
-python3 -m monoloco.run predict --webcam --scale 0.2 --output_types combined --z_max 10 --checkpoint resnet50 --model data/models/monoloco-190513-1437.pkl
+-o <output directory> \
+--long-edge <rescale the image by providing dimension of long side>
+--n_dropout <50 to include epistemic uncertainty, 0 otherwise>
 ```
 
-# Preprocessing
+![predict](docs/out_002282.png.multi.jpg)
 
-### Datasets
+To show all the instances estimated by MonoLoco add the argument `--show_all` to the above command.
 
-#### 1) KITTI dataset
+![predict_all](docs/out_002282.png.multi_all.jpg)
-Download KITTI ground truth files and camera calibration matrices for training
-from [here](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d) and
-save them respectively into `data/kitti/gt` and `data/kitti/calib`. 
-To extract pifpaf joints, you also need to download training images soft link the folder in `
-data/kitti/images`
 
-#### 2) nuScenes dataset
+It is also possible to run [openpifpaf](https://github.com/vita-epfl/openpifpaf) directly
-Download nuScenes dataset from [nuScenes](https://www.nuscenes.org/download) (either Mini or TrainVal), 
+by using `--mode keypoints`. All the other pifpaf arguments are also supported 
-save it anywhere and soft link it in `data/nuscenes`
+and can be checked with `python3 -m monoloco.run predict --help`.
 
-nuScenes preprocessing requires `pip3 install nuscenes-devkit`
+![predict](docs/out_002282_pifpaf.jpg)
 
 
-### Annotations to preprocess
+**Stereo Examples** <br /> 
-MonoLoco is trained using 2D human pose joints. To create them run pifaf over KITTI or nuScenes training images. 
+To run MonStereo on stereo images, make sure the stereo pairs have the following name structure:
-You can create them running the predict script and using `--networks pifpaf`.
+- Left image: \<name>.\<extension>
+- Right image: \<name>**_r**.\<extension>
 
-### Inputs joints for training
+(It does not matter the exact suffix as long as the images are ordered)
-MonoLoco is trained using 2D human pose joints matched with the ground truth location provided by
-nuScenes or KITTI Dataset. To create the joints run: `python3 -m monoloco.run prep` specifying:
-1. `--dir_ann` annotation directory containing Pifpaf joints of KITTI or nuScenes. 
 
-2. `--dataset` Which dataset to preprocess. For nuscenes, all three versions of the 
+You can load one or more image pairs using glob expressions. For example:
-dataset are supported: nuscenes_mini, nuscenes, nuscenes_teaser.
 
-### Ground truth file for evaluation
+```sh
-The preprocessing script also outputs a second json file called **names-<date-time>.json** which provide a dictionary indexed
+python3 -m monoloco.run predict --mode stereo \
-by the image name to easily access ground truth files for evaluation and prediction purposes.
+--glob docs/000840*.png
+ --path_gt <to match results with ground-truths> \
+ -o data/output  -long_edge 2500
+ ```
  
+![Crowded scene](docs/out_000840_multi.jpg)
 
+```sh
+python3 -m monoloco.run predict --glob docs/005523*.png \ --output_types multi \
+--mode stereo \
+--path_gt <to match results with ground-truths> \
+-o data/output  --long_edge 2500 \
+--instance-threshold 0.05 --seed-threshold 0.05
+ ```
+
+![Occluded hard example](docs/out_005523.png.multi.jpg)
+
+## B) Social Distancing (and Talking activity)
+To visualize social distancing compliance, simply add the argument `social_distance` to `--activities`. This visualization is not supported with a stereo camera.
+Threshold distance and radii (for F-formations) can be set using `--threshold-dist` and `--radii`, respectively.
+
+For more info, run:
+`python3 -m monoloco.run predict --help`
+
+**Examples** <br>
+An example from the Collective Activity Dataset is provided below.
+
+<img src="docs/frame0032.jpg" width="500"/>
+
+To visualize social distancing run the below, command:
+
+```sh
+pip3 install scipy
+```
+
+```sh
+python3 -m monoloco.run predict docs/frame0032.jpg \
+--activities social_distance --output_types front bird 
+```
+
+<img src="docs/out_frame0032_front_bird.jpg" width="700"/>
+
+## C) Hand-raising detection
+To detect raised hand, you can add the argument `--activities raise_hand` to the prediction command.
+
+For example, the below image is obtained with:
+```sh
+python3 -m monoloco.run predict docs/raising_hand.jpg \
+--activities raise_hand social_distance --output_types front
+```
+
+<img src="docs/out_raising_hand.jpg.front.jpg" width="500"/>
+
+For more info, run:
+`python3 -m monoloco.run predict --help`
+
+## D) Orientation and Bounding Box dimensions 
+The network estimates orientation and box dimensions as well. Results are saved in a json file when using the command 
+`--output_types json`. At the moment, the only visualization including orientation is the social distancing one.
+<br /> 
+
+## E) Webcam
+You can use the webcam as input by using the `--webcam` argument. By default the `--z_max` is set to 10 while using the webcam and the `--long-edge` is set to 144. If multiple webcams are plugged in you can choose between them using `--camera`, for instance to use the second camera you can add `--camera 1`.
+You also need to install `opencv-python` to use this feature :
+```sh
+pip3 install opencv-python
+```
+Example command:
+
+```sh
+python3 -m monoloco.run predict --webcam \
+--activities raise_hand social_distance
+```
 
 # Training
-Provide the json file containing the preprocess joints as argument. 
+We train on the KITTI dataset (MonoLoco/Monoloco++/MonStereo) or the nuScenes dataset (MonoLoco) specifying the path of the json file containing the input joints. Please download them [here](https://drive.google.com/drive/folders/1j0riwbS9zuEKQ_3oIs_dWlYBnfuN2WVN?usp=sharing) or follow [preprocessing instructions](#Preprocessing).
 
-As simple as `python3 -m monoloco.run --train --joints <json file path>`
+Results for [MonoLoco++](###Tables) are obtained with: 
 
-All the hyperparameters options can be checked at `python3 -m monoloco.run train --help`.
+```sh
+python3 -m monoloco.run train --joints data/arrays/joints-kitti-mono-210422-1600.json
+```
 
-### Hyperparameters tuning
+While for the [MonStereo](###Tables) results run:
-Random search in log space is provided. An example: `python3 -m monoloco.run train --hyp --multiplier 10 --r_seed 1`.
+
-One iteration of the multiplier includes 6 runs.
+```sh
+python3 -m monoloco.run train --joints data/arrays/joints-kitti-stereo-210422-1601.json \
+--lr 0.003 --mode stereo 
+```
+
+If you are interested in the original results of the MonoLoco ICCV article (now improved with MonoLoco++), please refer to the tag v0.4.9 in this repository.
+
+Finally, for a more extensive list of available parameters, run:
+
+`python3 -m monstereo.run train --help`
+
+<br /> 
+
+# Preprocessing
+Preprocessing and training step are already fully supported by the code provided, 
+but require first to run a pose detector over
+all the training images and collect the annotations. 
+The code supports this option (by running the predict script and using `--mode keypoints`).
+
+## Data structure
+
+    data         
+    ├── outputs                 
+    ├── arrays
+    ├── kitti
+    
+Run the following inside monoloco repository:
+```
+mkdir data
+cd data
+mkdir outputs arrays kitti
+```
 
 
-# Evaluation (KITTI Dataset)
+## Kitti Dataset
-We provide evaluation on KITTI for models trained on nuScenes or KITTI. We compare them with other monocular 
+Download kitti images (from left and right cameras), ground-truth files (labels), and calibration files from their [website](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d) and save them inside the `data` folder as shown below.
-and stereo Baselines: 
+
+    data         
+    ├── kitti
+            ├── gt
+            ├── calib
+            ├── images
+            ├── images_right
+
+
+The network takes as inputs 2D keypoints annotations. To create them run PifPaf over the saved images:
+
+```sh
+python3 -m openpifpaf.predict \
+--glob "data/kitti/images/*.png" \
+--json-output <directory to contain predictions> \
+--checkpoint=shufflenetv2k30 \
+--instance-threshold=0.05 --seed-threshold 0.05 --force-complete-pose 
+```
+
+**Horizontal flipping**
+
+To augment the dataset, we apply horizontal flipping on the detected poses. To include small variations in the pose, we use the poses from the right-camera (the dataset uses a stereo camera). As there are no labels for the right camera, the code automatically correct the ground truth depth by taking into account the camera baseline.
+To obtain these poses, run pifpaf also on the folder of right images. Make sure to save annotations into a different folder, and call the right folder: `<NameOfTheLeftFolder>_right`
+
+**Recall**
+
+To maximize the recall (at the cost of the computational time), it's possible to upscale the images with the command `--long_edge 2500` (\~scale 2). 
+
+Once this step is complete, the below commands transform all the annotations into a single json file that will used for training.
+
+
+
+For MonoLoco++:
+```sh
+python3 -m monoloco.run prep --dir_ann <directory that contains annotations>
+```
+
+For MonStereo:
+```sh
+python3 -m monoloco.run prep --mode stereo --dir_ann <directory that contains left annotations> 
+```
+
+## Collective Activity Dataset
+To evaluate on of the [collective activity dataset](http://vhosts.eecs.umich.edu/vision//activity-dataset.html)
+ (without any training) we selected 6 scenes that contain people talking to each other. 
+ This allows for a balanced dataset, but any other configuration will work. 
+
+THe expected structure for the dataset is the following:
+
+    collective_activity         
+    ├── images                 
+    ├── annotations
+    
+where images and annotations inside have the following name convention:
+
+IMAGES: seq<sequence_name>_frame<frame_name>.jpg
+ANNOTATIONS: seq<sequence_name>_annotations.txt
+
+With respect to the original dataset, the images and annotations are moved to a single folder 
+and the sequence is added in their name. One command to do this is:
+
+`rename -v -n 's/frame/seq14_frame/'  f*.jpg`
+
+which for example change the name of all the jpg images in that folder adding the sequence number
+ (remove `-n` after checking it works)
+
+Pifpaf annotations should also be saved in a single folder and can be created with:
+
+```sh
+python3 -m openpifpaf.predict \
+--glob "data/collective_activity/images/*.jpg"  \
+--checkpoint=shufflenetv2k30 \
+--instance-threshold=0.05 --seed-threshold 0.05 \--force-complete-pose \
+--json-output <output folder>
+```
+
+# CASR dataset
+To train monoloco on the CASR dataset, we must first create the joints file by preprocessing the CASR annotations.
+To do this we create the following folder structure :
+
+    data         
+    ├── casr
+            ├── annotations
+            ├── models
+            ├── outputs
+            
+We then run monoloco on the images of the dataset and save the resulting annotations in a folder that we will call `<dir_ann>`.
+Then we can run :
+```sh
+python3 -m monoloco.run prep --dataset casr --dir_ann <dir_ann>
+```
+Which will create a joints file in `data/casr/outputs`. This file can be inputed into the trainer with `--mode casr` to train a model to recognize cyclist intention.
+```sh
+python3 -m monoloco.run train --mode casr --joints data/outputs/<joints_file>
+```
+This command can also be ran with hyperparamter tuning by adding the flag `--hyp`.
+
+To train a model to recognize only standard gestures from CASR, we can run the following commands :
+```sh
+python3 -m monoloco.run prep --dataset casr --casr_std --dir_ann <dir_ann>
+python3 -m monoloco.run train --mode casr_std --joints data/outputs/<joints_file>
+```
+Once we have obtained a trained model, we can predict cyclist intention by using the following command :
+```sh
+python3 -m monoloco.run predict \
+--glob <path_to_images> \
+--casr --activities is_turning \
+--casr_model data/models/<trained_model>
+```
+Or this one for only standard gestures:
+```sh
+python3 -m monoloco.run predict \
+--glob <path_to_images> \
+--casr_std --activities is_turning \
+--casr_model data/models/<trained_model>
+```
+
+# Evaluation
+
+## 3D Localization
+We provide evaluation on KITTI for models trained on nuScenes or KITTI. Download the ground-truths of KITTI dataset and the calibration files from their [website](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d). Save the training labels (one .txt file for each image) into the folder `data/kitti/gt` and the camera calibration matrices (one .txt file for each image) into `data/kitti/calib`.  
+To evaluate a pre-trained model, download the latest models from [here](https://drive.google.com/drive/u/0/folders/1kQpaTcDsiNyY6eh1kUurcpptfAXkBjAJ) and save them into `data/outputs.
+
+__Baselines__
+
+We compare our results with other monocular 
+and stereo baselines, depending whether you are evaluating stereo or monocular settings. For some of the baselines, we have obtained the annotations directly from the authors and we don't have yet the permission to publish them. 
 
 [Mono3D](https://www.cs.toronto.edu/~urtasun/publications/chen_etal_cvpr16.pdf), 
 [3DOP](https://xiaozhichen.github.io/papers/nips15chen.pdf), 
-[MonoDepth](https://arxiv.org/abs/1609.03677) and our 
+[MonoDepth](https://arxiv.org/abs/1609.03677) 
+[MonoPSR](https://github.com/kujason/monopsr) and our 
+[MonoDIS](https://research.mapillary.com/img/publications/MonoDIS.pdf) and our 
 [Geometrical Baseline](monoloco/eval/geom_baseline.py).
 
 * **Mono3D**: download validation files from [here](http://3dimage.ee.tsinghua.edu.cn/cxz/mono3d) 
@@ -207,17 +419,76 @@ and save them into `data/kitti/3dop`
 * **MonoDepth**: compute an average depth for every instance using the following script 
 [here](https://github.com/Parrotlife/pedestrianDepth-baseline/tree/master/MonoDepth-PyTorch) 
 and save them into `data/kitti/monodepth`
-* **GeometricalBaseline**: A geometrical baseline comparison is provided. 
+* **Geometrical Baseline and MonoLoco**:
-The average geometrical value for comparison can be obtained running:
+To include also geometric baselines and MonoLoco, download a monoloco model, save it in `data/models`, and add the flag ``--baselines`` to the evaluation command
-`python3 -m monoloco.run eval --geometric 
---model data/models/monoloco-190719-0923.pkl --joints data/arrays/joints-nuscenes_teaser-190717-1424.json`
 
 
-The following results are obtained running:
+The evaluation file will run the model over all the annotations and compare the results with KITTI  ground-truth and the downloaded baselines. For this run:
-`python3 -m monoloco.run eval --model data/models/monoloco-190719-0923.pkl --generate
---dir_ann <folder containing pifpaf annotations of KITTI images>`
 
-![kitti_evaluation](docs/results.png)
+```sh
-![kitti_evaluation_table](docs/results_table.png)
+python3 -m monoloco.run eval \
+--dir_ann <annotation directory> \
+--model data/outputs/monoloco_pp-210422-1601.pkl \
+--generate \
+--save \
+```
+
+For stereo results add `--mode stereo` and select `--model=monstereo-210422-1620.pkl`.  Below, the resulting table of results and an example of the saved figures.
+
+## Tables
+
+<img src="docs/quantitative.jpg" width="700"/>
+
+<img src="docs/results_monstereo.jpg" width="700"/>
 
 
+## Relative Average Precision Localization: RALP-5% (MonStereo)
+
+We modified the original C++ evaluation of KITTI to make it relative to distance. We use **cmake**.
+To run the evaluation, first generate the txt file with the standard command for evaluation (above).
+Then follow the instructions of this [repository](https://github.com/cguindel/eval_kitti) 
+to prepare the folders accordingly (or follow kitti guidelines) and run evaluation. 
+The modified file is called *evaluate_object.cpp* and runs exactly as the original kitti evaluation.
+
+## Activity Estimation (Talking)
+Please follow preprocessing steps for Collective activity dataset and run pifpaf over the dataset images.
+Evaluation on this dataset is done with models trained on either KITTI or nuScenes. 
+For optimal performances, we suggest the model trained on nuScenes teaser.
+
+```sh
+python3 -m monstereo.run eval \
+--activity \
+--dataset collective \
+--model <path to the model> \
+--dir_ann <annotation directory>
+```
+
+# Citation
+When using this library in your research, we will be happy if you cite us! 
+
+```
+@InProceedings{bertoni_2021_icra,
+    author = {Bertoni, Lorenzo and Kreiss, Sven and Mordan, Taylor and Alahi, Alexandre},
+    title = {MonStereo: When Monocular and Stereo Meet at the Tail of 3D Human Localization},
+    booktitle = {the International Conference on Robotics and Automation (ICRA)},
+    year = {2021}
+}
+```
+
+```
+@ARTICLE{bertoni_2021_its,
+    author = {Bertoni, Lorenzo and Kreiss, Sven and Alahi, Alexandre},
+    journal={IEEE Transactions on Intelligent Transportation Systems}, 
+    title={Perceiving Humans: from Monocular 3D Localization to Social Distancing}, 
+    year={2021},
+```
+
+```
+@InProceedings{bertoni_2019_iccv,
+    author = {Bertoni, Lorenzo and Kreiss, Sven and Alahi, Alexandre},
+    title = {MonoLoco: Monocular 3D Pedestrian Localization and Uncertainty Estimation},
+    booktitle = {the IEEE International Conference on Computer Vision (ICCV)},
+    month = {October},
+    year = {2019}
+}
+```

kitti-eval/CMakeLists.txt

@@ -0,0 +1,8 @@
+cmake_minimum_required (VERSION 2.6)
+project(devkit_object)
+
+find_package(PNG REQUIRED)
+
+add_executable(evaluate_object evaluate_object.cpp)
+include_directories(${PNG_INCLUDE_DIR})
+target_link_libraries(evaluate_object ${PNG_LIBRARY})

kitti-eval/README.md

@@ -0,0 +1,34 @@
+# eval_kitti #
+
+[![Build Status](https://travis-ci.org/cguindel/eval_kitti.svg?branch=master)](https://travis-ci.org/cguindel/eval_kitti)
+[![License: CC BY-NC-SA](https://img.shields.io/badge/License-CC%20BY--NC--SA%203.0-lightgrey.svg)](https://creativecommons.org/licenses/by-nc-sa/3.0/)
+
+The *eval_kitti* software contains tools to evaluate object detection results using the KITTI dataset. The code is based on the [KITTI object development kit](http://www.cvlibs.net/datasets/kitti/eval_object.php).
+
+### Tools ###
+
+* *evaluate_object* is an improved version of the official KITTI evaluation that enables multi-class evaluation and splits of the training set for validation. It's updated according to the modifications introduced in 2017 by the KITTI authors.
+* *parser* is meant to provide mAP and mAOS stats from the precision-recall curves obtained with the evaluation script.
+* *create_link* is a helper that can be used to create a link to the results obtained with [lsi-faster-rcnn](https://github.com/cguindel/lsi-faster-rcnn).
+
+### Usage ###
+Build *evaluate_object* with CMake:
+```
+mkdir build
+cd build
+cmake ..
+make
+```
+
+The `evaluate_object` executable will be then created inside `build`. The following folders are also required to be placed there in order to perform the evaluation:
+
+* `data/object/label_2`, with the KITTI dataset labels.
+* `lists`, containing the  `.txt` files with the train/validation splits. These files are expected to contain a list of the used image indices, one per row.
+* `results`, in which a subfolder should be created for every test, including a second-level `data` folder with the resulting `.txt` files to be evaluated.
+
+`evaluate_object` should be called with the name of the results folder and the validation split; e.g.: ```./evaluate_object leaderboard valsplit ```
+
+`parser` needs the results folder; e.g.: ```./parser.py leaderboard ```. **Note**: *parser* will only provide results for *Car*, *Pedestrian* and *Cyclist*; modify it (line 8) if you need to evaluate the rest of classes.  
+
+### Copyright ###
+This work is a derivative of [The KITTI Vision Benchmark Suite](http://www.cvlibs.net/datasets/kitti/eval_object.php) by A. Geiger, P. Lenz, C. Stiller and R. Urtasun, used under [CC BY-NC-SA](https://creativecommons.org/licenses/by-nc-sa/3.0/). Consequently, code in this repository is published under the same [Creative Commons Attribution-NonCommercial-ShareAlike 3.0 License](https://creativecommons.org/licenses/by-nc-sa/3.0/). This means that you must attribute the work in the manner specified by the authors, you may not use this work for commercial purposes and if you alter, transform, or build upon this work, you may distribute the resulting work only under the same license.

kitti-eval/parser.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import numpy as np
+
+# CLASSES = ['car', 'pedestrian', 'cyclist', 'van', 'truck', 'person_sitting', 'tram']
+CLASSES = ['pedestrian']
+
+# PARAMS = ['detection', 'orientation', 'iour', 'mppe']
+PARAMS = ['detection', 'detection_1%', 'detection_5%', 'detection_10%', 'detection_3d', 'detection_ground', 'orientation']
+
+DIFFICULTIES = ['easy', 'moderate', 'hard', 'all']
+
+eval_type = ''
+
+if len(sys.argv)<2:
+    print('Usage: parser.py results_folder [evaluation_type]')
+
+if len(sys.argv)==3:
+    eval_type = sys.argv[2]
+
+result_sha = sys.argv[1]
+txt_dir = os.path.join('build','results', result_sha)
+
+for class_name in CLASSES:
+    for param in PARAMS:
+        print("--{:s} {:s}--".format(class_name, param))
+        if eval_type is '':
+            txt_name = os.path.join(txt_dir, 'stats_' + class_name + '_' + param + '.txt')
+        else:
+            txt_name = os.path.join(txt_dir, 'stats_' + class_name + '_' + param + '_' + eval_type + '.txt')
+
+        if not os.path.isfile(txt_name):
+            print(txt_name, ' not found')
+            continue
+
+        cont = np.loadtxt(txt_name)
+
+        averages = []
+        for idx, difficulty in enumerate(DIFFICULTIES):
+            sum = 0
+            if param in PARAMS:
+                for i in range(1, 41):
+                    sum += cont[idx][i]
+
+                average = sum/40.0
+                
+            #print class_name, difficulty, param, average
+            averages.append(average)
+
+        #print "\n"+class_name+" "+param
+        print("Easy\tMod.\tHard\tAll")
+        print("{:.2f}\t{:.2f}\t{:.2f}\t{:.2f}".format(100*averages[0], 100*averages[1],100*averages[2],100*averages[3]))
+        print("---------------------------------------------------------------------------------")
+        if eval_type is not '' and param=='detection':
+            break # No orientation for 3D or bird eye
+
+    #print '================='

monoloco/__init__.py

@@ -1,4 +1,8 @@
 
-"""Open implementation of MonoLoco."""
+"""
+Open implementation of MonoLoco / MonoLoco++ / MonStereo
+"""
 
-__version__ = '0.4.9'
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions

monoloco/_version.py

@@ -0,0 +1,527 @@
+
+# This file helps to compute a version number in source trees obtained from
+# git-archive tarball (such as those provided by githubs download-from-tag
+# feature). Distribution tarballs (built by setup.py sdist) and build
+# directories (produced by setup.py build) will contain a much shorter file
+# that just contains the computed version number.
+
+# This file is released into the public domain. Generated by
+# versioneer-0.19 (https://github.com/python-versioneer/python-versioneer)
+
+# pylint: skip-file
+
+"""Git implementation of _version.py."""
+
+import errno
+import os
+import re
+import subprocess
+import sys
+
+
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "$Format:%d$"
+    git_full = "$Format:%H$"
+    git_date = "$Format:%ci$"
+    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = "v"
+    cfg.parentdir_prefix = "None"
+    cfg.versionfile_source = "monoloco/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Create decorator to mark a method as the handler of a VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+                env=None):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+                                 stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None, None
+    else:
+        if verbose:
+            print("unable to find command, tried %s" % (commands,))
+        return None, None
+    stdout = p.communicate()[0].strip().decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % dispcmd)
+            print("stdout was %s" % stdout)
+        return None, p.returncode
+    return stdout, p.returncode
+
+
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes both
+    the project name and a version string. We will also support searching up
+    two directory levels for an appropriately named parent directory
+    """
+    rootdirs = []
+
+    for i in range(3):
+        dirname = os.path.basename(root)
+        if dirname.startswith(parentdir_prefix):
+            return {"version": dirname[len(parentdir_prefix):],
+                    "full-revisionid": None,
+                    "dirty": False, "error": None, "date": None}
+        else:
+            rootdirs.append(root)
+            root = os.path.dirname(root)  # up a level
+
+    if verbose:
+        print("Tried directories %s but none started with prefix %s" %
+              (str(rootdirs), parentdir_prefix))
+    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
+    # the code embedded in _version.py can just fetch the value of these
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
+    try:
+        f = open(versionfile_abs, "r")
+        for line in f.readlines():
+            if line.strip().startswith("git_refnames ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["refnames"] = mo.group(1)
+            if line.strip().startswith("git_full ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["full"] = mo.group(1)
+            if line.strip().startswith("git_date ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["date"] = mo.group(1)
+        f.close()
+    except EnvironmentError:
+        pass
+    return keywords
+
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    date = keywords.get("date")
+    if date is not None:
+        # Use only the last line.  Previous lines may contain GPG signature
+        # information.
+        date = date.splitlines()[-1]
+
+        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
+        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
+        # -like" string, which we must then edit to make compliant), because
+        # it's been around since git-1.5.3, and it's too difficult to
+        # discover which version we're using, or to work around using an
+        # older one.
+        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+    refnames = keywords["refnames"].strip()
+    if refnames.startswith("$Format"):
+        if verbose:
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
+    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
+    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
+    TAG = "tag: "
+    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+    if not tags:
+        # Either we're using git < 1.8.3, or there really are no tags. We use
+        # a heuristic: assume all version tags have a digit. The old git %d
+        # expansion behaves like git log --decorate=short and strips out the
+        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
+        # between branches and tags. By ignoring refnames without digits, we
+        # filter out many common branch names like "release" and
+        # "stabilization", as well as "HEAD" and "master".
+        tags = set([r for r in refs if re.search(r'\d', r)])
+        if verbose:
+            print("discarding '%s', no digits" % ",".join(refs - tags))
+    if verbose:
+        print("likely tags: %s" % ",".join(sorted(tags)))
+    for ref in sorted(tags):
+        # sorting will prefer e.g. "2.0" over "2.0rc1"
+        if ref.startswith(tag_prefix):
+            r = ref[len(tag_prefix):]
+            if verbose:
+                print("picking %s" % r)
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None,
+                    "date": date}
+    # no suitable tags, so version is "0+unknown", but full hex is still there
+    if verbose:
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags", "date": None}
+
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    GITS = ["git"]
+    if sys.platform == "win32":
+        GITS = ["git.cmd", "git.exe"]
+
+    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+                          hide_stderr=True)
+    if rc != 0:
+        if verbose:
+            print("Directory %s not under git control" % root)
+        raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+                                          "--always", "--long",
+                                          "--match", "%s*" % tag_prefix],
+                                   cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                    cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    # commit date: see ISO-8601 comment in git_versions_from_keywords()
+    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
+                       cwd=root)[0].strip()
+    # Use only the last line.  Previous lines may contain GPG signature
+    # information.
+    date = date.splitlines()[-1]
+    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post0.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post0.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post0.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post0.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"],
+                "date": None}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None,
+            "date": pieces.get("date")}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
+        # versionfile_source is the relative path from the top of the source
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
+            root = os.path.dirname(root)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree",
+                "date": None}
+
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version", "date": None}

monoloco/activity.py

@@ -0,0 +1,361 @@
+
+# pylint: disable=too-many-statements
+
+import math
+import copy
+from contextlib import contextmanager
+
+import numpy as np
+import torch
+import matplotlib.pyplot as plt
+
+from .network.process import laplace_sampling
+from .visuals.pifpaf_show import (
+    KeypointPainter, image_canvas, get_pifpaf_outputs, draw_orientation, social_distance_colors
+)
+
+
+def social_interactions(idx, centers, angles, dds, stds=None, social_distance=False,
+                        n_samples=100, threshold_prob=0.25, threshold_dist=2, radii=(0.3, 0.5)):
+    """
+    return flag of alert if social distancing is violated
+    """
+
+    # A) Check whether people are close together
+    xx = centers[idx][0]
+    zz = centers[idx][1]
+    distances = [math.sqrt((xx - centers[i][0]) ** 2 + (zz - centers[i][1]) ** 2)
+                 for i, _ in enumerate(centers)]
+    sorted_idxs = np.argsort(distances)
+    indices = [idx_t for idx_t in sorted_idxs[1:]
+               if distances[idx_t] <= threshold_dist]
+
+    # B) Check whether people are looking inwards and whether there are no intrusions
+    # Deterministic
+    if n_samples < 2:
+        for idx_t in indices:
+            if check_f_formations(idx, idx_t, centers, angles,
+                                  radii=radii,  # Binary value
+                                  social_distance=social_distance):
+                return True
+
+    # Probabilistic
+    else:
+        # Samples distance
+        dds = torch.tensor(dds).view(-1, 1)
+        stds = torch.tensor(stds).view(-1, 1)
+        # stds_te = get_task_error(dds)  # similar results to MonoLoco but lower true positive
+        laplace_d = torch.cat((dds, stds), dim=1)
+        samples_d = laplace_sampling(laplace_d, n_samples=n_samples)
+
+        # Iterate over close people
+        for idx_t in indices:
+            f_forms = []
+            for s_d in range(n_samples):
+                new_centers = copy.deepcopy(centers)
+                for el in (idx, idx_t):
+                    delta_d = dds[el] - float(samples_d[s_d, el])
+                    theta = math.atan2(new_centers[el][1], new_centers[el][0])
+                    delta_x = delta_d * math.cos(theta)
+                    delta_z = delta_d * math.sin(theta)
+                    new_centers[el][0] += delta_x
+                    new_centers[el][1] += delta_z
+                f_forms.append(check_f_formations(idx, idx_t, new_centers, angles,
+                                                  radii=radii,
+                                                  social_distance=social_distance))
+            if (sum(f_forms) / n_samples) >= threshold_prob:
+                return True
+    return False
+
+
+def is_turning(kp):
+    """
+    Returns flag if a cyclist is turning
+    """
+    x=0
+    y=1
+
+    nose = 0
+    l_ear = 3
+    r_ear = 4
+    l_shoulder = 5
+    l_elbow = 7
+    l_hand = 9
+    r_shoulder = 6
+    r_elbow = 8
+    r_hand = 10
+
+    head_width = kp[x][l_ear]- kp[x][r_ear]
+    head_top = (kp[y][nose] - head_width)
+
+    l_forearm = [kp[x][l_hand] - kp[x][l_elbow], kp[y][l_hand] - kp[y][l_elbow]]
+    l_arm = [kp[x][l_shoulder] - kp[x][l_elbow], kp[y][l_shoulder] - kp[y][l_elbow]]
+
+    r_forearm = [kp[x][r_hand] - kp[x][r_elbow], kp[y][r_hand] - kp[y][r_elbow]]
+    r_arm = [kp[x][r_shoulder] - kp[x][r_elbow], kp[y][r_shoulder] - kp[y][r_elbow]]
+
+    l_angle = (90/np.pi) * np.arccos(np.dot(l_forearm/np.linalg.norm(l_forearm), l_arm/np.linalg.norm(l_arm)))
+    r_angle = (90/np.pi) * np.arccos(np.dot(r_forearm/np.linalg.norm(r_forearm), r_arm/np.linalg.norm(r_arm)))
+
+    if kp[x][l_shoulder] > kp[x][r_shoulder]:
+        is_left = kp[x][l_hand] > kp[x][l_shoulder] + np.linalg.norm(l_arm)
+        is_right = kp[x][r_hand] < kp[x][r_shoulder] - np.linalg.norm(r_arm)
+        l_too_close = kp[x][l_hand] > kp[x][l_shoulder] and kp[y][l_hand]>=head_top
+        r_too_close = kp[x][r_hand] < kp[x][r_shoulder] and kp[y][r_hand]>=head_top
+    else:
+        is_left = kp[x][l_hand] < kp[x][l_shoulder] - np.linalg.norm(l_arm)
+        is_right = kp[x][r_hand] > kp[x][r_shoulder] + np.linalg.norm(r_arm)
+        l_too_close = kp[x][l_hand] <= kp[x][l_shoulder] and kp[y][l_hand]>=head_top
+        r_too_close = kp[x][r_hand] >= kp[x][r_shoulder] and kp[y][r_hand]>=head_top
+
+
+    is_l_up = kp[y][l_hand] < kp[y][l_shoulder]
+    is_r_up = kp[y][r_hand] < kp[y][r_shoulder]
+
+    is_left_risen = is_l_up and l_angle >= 30 and not l_too_close
+    is_right_risen = is_r_up and r_angle >= 30 and not r_too_close
+
+    is_left_down = is_l_up and l_angle >= 30 and not l_too_close
+    is_right_down = is_r_up and r_angle >= 30 and not r_too_close
+
+    if is_left and l_angle >= 40 and not(is_left_risen or is_right_risen):
+        return 'left'
+
+    if is_right and r_angle >= 40 or (is_left_risen or is_right_risen):
+        return 'right'
+
+    if is_left_down or is_right_down:
+        return 'stop'
+
+    return None
+
+
+def is_phoning(kp):
+    """
+    Returns flag of alert if someone is using their phone
+    """
+    x=0
+    y=1
+
+    nose = 0
+    l_ear = 3
+    l_shoulder = 5
+    l_elbow = 7
+    l_hand = 9
+    r_ear = 4
+    r_shoulder = 6
+    r_elbow = 8
+    r_hand = 10
+
+    head_width = kp[x][l_ear]- kp[x][r_ear]
+    head_top = (kp[y][nose] - head_width)
+
+    l_forearm = [kp[x][l_hand] - kp[x][l_elbow], kp[y][l_hand] - kp[y][l_elbow]]
+    l_arm = [kp[x][l_shoulder] - kp[x][l_elbow], kp[y][l_shoulder] - kp[y][l_elbow]]
+
+    r_forearm = [kp[x][r_hand] - kp[x][r_elbow], kp[y][r_hand] - kp[y][r_elbow]]
+    r_arm = [kp[x][r_shoulder] - kp[x][r_elbow], kp[y][r_shoulder] - kp[y][r_elbow]]
+
+    l_angle = (90/np.pi) * np.arccos(np.dot(l_forearm/np.linalg.norm(l_forearm), l_arm/np.linalg.norm(l_arm)))
+    r_angle = (90/np.pi) * np.arccos(np.dot(r_forearm/np.linalg.norm(r_forearm), r_arm/np.linalg.norm(r_arm)))
+
+    is_l_up = kp[y][l_hand] < kp[y][l_shoulder]
+    is_r_up = kp[y][r_hand] < kp[y][r_shoulder]
+
+    l_too_close = kp[x][l_hand] <= kp[x][l_shoulder] and kp[y][l_hand]>=head_top
+    r_too_close = kp[x][r_hand] >= kp[x][r_shoulder] and kp[y][r_hand]>=head_top
+
+    is_left_phone = is_l_up and l_angle <= 30 and l_too_close
+    is_right_phone = is_r_up and r_angle <= 30 and r_too_close
+
+    print("Top of head y is :", head_top)
+    print("Nose height :", kp[y][nose])
+    print("Right elbow x: {} and y: {}".format(kp[x][r_elbow], kp[y][r_elbow]))
+    print("Left elbow x: {} and y: {}".format(kp[x][l_elbow], kp[y][l_elbow]))
+
+    print("Right shoulder height :", kp[y][r_shoulder])
+    print("Left shoulder height :", kp[y][l_shoulder])
+
+    print("Left hand x = ", kp[x][l_hand])
+    print("Left hand y = ", kp[y][l_hand])
+
+    print("Is left hand up : ", is_l_up)
+
+    print("Right hand x = ", kp[x][r_hand])
+    print("Right hand y = ", kp[y][r_hand])
+
+    print("Is right hand up : ", is_r_up)
+
+    print("Left arm angle :", l_angle)
+    print("Right arm angle :", r_angle)
+
+    print("Is left hand close to head :", l_too_close)
+    print("Is right hand close to head:", r_too_close)
+
+    if is_left_phone or is_right_phone:
+        return True
+
+    return False
+
+
+def is_raising_hand(kp):
+    """
+    Returns flag of alert if someone raises their hand
+    """
+    x=0
+    y=1
+
+    nose = 0
+    l_ear = 3
+    l_shoulder = 5
+    l_elbow = 7
+    l_hand = 9
+    r_ear = 4
+    r_shoulder = 6
+    r_elbow = 8
+    r_hand = 10
+
+    head_width = kp[x][l_ear]- kp[x][r_ear]
+    head_top = (kp[y][nose] - head_width)
+
+    l_forearm = [kp[x][l_hand] - kp[x][l_elbow], kp[y][l_hand] - kp[y][l_elbow]]
+    l_arm = [kp[x][l_shoulder] - kp[x][l_elbow], kp[y][l_shoulder] - kp[y][l_elbow]]
+
+    r_forearm = [kp[x][r_hand] - kp[x][r_elbow], kp[y][r_hand] - kp[y][r_elbow]]
+    r_arm = [kp[x][r_shoulder] - kp[x][r_elbow], kp[y][r_shoulder] - kp[y][r_elbow]]
+
+    l_angle = (90/np.pi) * np.arccos(np.dot(l_forearm/np.linalg.norm(l_forearm), l_arm/np.linalg.norm(l_arm)))
+    r_angle = (90/np.pi) * np.arccos(np.dot(r_forearm/np.linalg.norm(r_forearm), r_arm/np.linalg.norm(r_arm)))
+
+    is_l_up = kp[y][l_hand] < kp[y][l_shoulder]
+    is_r_up = kp[y][r_hand] < kp[y][r_shoulder]
+
+    l_too_close = kp[x][l_hand] <= kp[x][l_shoulder] and kp[y][l_hand]>=head_top
+    r_too_close = kp[x][r_hand] >= kp[x][r_shoulder] and kp[y][r_hand]>=head_top
+
+    is_left_risen = is_l_up and l_angle >= 30 and not l_too_close
+    is_right_risen = is_r_up and r_angle >= 30 and not r_too_close
+
+    if is_left_risen and is_right_risen:
+        return 'both'
+
+    if is_left_risen:
+        return 'left'
+
+    if is_right_risen:
+        return 'right'
+
+    return None
+
+
+def check_f_formations(idx, idx_t, centers, angles, radii, social_distance=False):
+    """
+    Check F-formations for people close together (this function do not expect far away people):
+    1) Empty space of a certain radius (no other people or themselves inside)
+    2) People looking inward
+    """
+
+    # Extract centers and angles
+    other_centers = np.array(
+        [cent for l, cent in enumerate(centers) if l not in (idx, idx_t)])
+    theta0 = angles[idx]
+    theta1 = angles[idx_t]
+
+    # Find the center of o-space as average of two candidates (based on their orientation)
+    for radius in radii:
+        x_0 = np.array([float(centers[idx][0]), float(centers[idx][1])])
+        x_1 = np.array([float(centers[idx_t][0]), float(centers[idx_t][1])])
+
+        mu_0 = np.array([
+            float(centers[idx][0]) + radius * math.cos(theta0),
+            float(centers[idx][1]) - radius * math.sin(theta0)])
+        mu_1 = np.array([
+            float(centers[idx_t][0]) + radius * math.cos(theta1),
+            float(centers[idx_t][1]) - radius * math.sin(theta1)])
+        o_c = (mu_0 + mu_1) / 2
+
+        # 1) Verify they are looking inwards.
+        # The distance between mus and the center should be less wrt the original position and the center
+        d_new = np.linalg.norm(
+            mu_0 - mu_1) / 2 if social_distance else np.linalg.norm(mu_0 - mu_1)
+        d_0 = np.linalg.norm(x_0 - o_c)
+        d_1 = np.linalg.norm(x_1 - o_c)
+
+        # 2) Verify no intrusion for third parties
+        if other_centers.size:
+            other_distances = np.linalg.norm(
+                other_centers - o_c.reshape(1, -1), axis=1)
+        else:
+            # Condition verified if no other people
+            other_distances = 100 * np.ones((1, 1))
+
+        # Binary Classification
+        # if np.min(other_distances) > radius:  # Ablation without orientation
+        if d_new <= min(d_0, d_1) and np.min(other_distances) > radius:
+            return True
+    return False
+
+
+def show_activities(args, image_t, output_path, annotations, dic_out):
+    """Output frontal image with poses or combined with bird eye view"""
+
+    assert 'front' in args.output_types or 'bird' in args.output_types, "outputs allowed: front and/or bird"
+
+    colors = ['deepskyblue' for _ in dic_out['uv_heads']]
+    if 'social_distance' in args.activities:
+        colors = social_distance_colors(colors, dic_out)
+
+    angles = dic_out['angles']
+    stds = dic_out['stds_ale']
+    xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']]
+
+    # Draw keypoints and orientation
+    if 'front' in args.output_types:
+        keypoint_sets, _ = get_pifpaf_outputs(annotations)
+        uv_centers = dic_out['uv_heads']
+        sizes = [abs(dic_out['uv_heads'][idx][1] - uv_s[1]) / 1.5 for idx, uv_s in
+                 enumerate(dic_out['uv_shoulders'])]
+        keypoint_painter = KeypointPainter(show_box=False)
+
+        with image_canvas(image_t,
+                          output_path + '.front.png',
+                          show=args.show,
+                          fig_width=10,
+                          dpi_factor=1.0) as ax:
+            keypoint_painter.keypoints(
+                ax, keypoint_sets, activities=args.activities, dic_out=dic_out,
+                size=image_t.size, colors=colors)
+            draw_orientation(ax, uv_centers, sizes,
+                             angles, colors, mode='front')
+
+    if 'bird' in args.output_types:
+        z_max = min(args.z_max, 4 + max([el[1] for el in xz_centers]))
+        with bird_canvas(output_path, z_max) as ax1:
+            draw_orientation(ax1, xz_centers, [], angles, colors, mode='bird')
+            draw_uncertainty(ax1, xz_centers, stds)
+
+
+@contextmanager
+def bird_canvas(output_path, z_max):
+    fig, ax = plt.subplots(1, 1)
+    fig.set_tight_layout(True)
+    output_path = output_path + '.bird.png'
+    x_max = z_max / 1.5
+    ax.plot([0, x_max], [0, z_max], 'k--')
+    ax.plot([0, -x_max], [0, z_max], 'k--')
+    ax.set_ylim(0, z_max + 1)
+    yield ax
+    fig.savefig(output_path)
+    plt.close(fig)
+    print('Bird-eye-view image saved')
+
+
+def draw_uncertainty(ax, centers, stds):
+    for idx, std in enumerate(stds):
+        std = stds[idx]
+        theta = math.atan2(centers[idx][1], centers[idx][0])
+        delta_x = std * math.cos(theta)
+        delta_z = std * math.sin(theta)
+        x = (centers[idx][0] - delta_x, centers[idx][0] + delta_x)
+        z = (centers[idx][1] - delta_z, centers[idx][1] + delta_z)
+        ax.plot(x, z, color='g', linewidth=2.5)

monoloco/eval/__init__.py

@@ -1,4 +1,2 @@
 
 from .eval_kitti import EvalKitti
-from .generate_kitti import GenerateKitti
-from .geom_baseline import geometric_baseline

monoloco/eval/eval_activity.py

@@ -0,0 +1,271 @@
+
+import os
+import glob
+import csv
+import copy
+from collections import defaultdict
+
+import numpy as np
+import torch
+from PIL import Image
+try:
+    from sklearn.metrics import accuracy_score
+    ACCURACY_SCORE = copy.copy(accuracy_score)
+except ImportError:
+    ACCURACY_SCORE = None
+
+from ..prep import factory_file
+from ..network import Loco
+from ..network.process import factory_for_gt, preprocess_pifpaf
+from ..activity import social_interactions
+from ..utils import open_annotations, get_iou_matches, get_difficulty
+
+
+class ActivityEvaluator:
+    """Evaluate talking activity for Collective Activity Dataset & KITTI"""
+
+    dic_cnt = dict(fp=0, fn=0, det=0)
+    cnt = {'pred': defaultdict(int), 'gt': defaultdict(int)}  # pred is for matched instances
+
+    def __init__(self, args):
+
+        self.dir_ann = args.dir_ann
+        assert self.dir_ann is not None and os.path.exists(self.dir_ann), \
+            "Annotation directory not provided / does not exist"
+        assert os.listdir(self.dir_ann), "Annotation directory is empty"
+
+        # COLLECTIVE ACTIVITY DATASET (talking)
+        # -------------------------------------------------------------------------------------------------------------
+        if args.dataset == 'collective':
+            self.sequences = ['seq02', 'seq14', 'seq12', 'seq13', 'seq11', 'seq36']
+            # folders_collective = ['seq02']
+            self.dir_data = 'data/activity/dataset'
+            self.THRESHOLD_PROB = 0.25  # Concordance for samples
+            self.THRESHOLD_DIST = 2  # Threshold to check distance of people
+            self.RADII = (0.3, 0.5)  # expected radii of the o-space
+            self.PIFPAF_CONF = 0.3
+            self.SOCIAL_DISTANCE = False
+        # -------------------------------------------------------------------------------------------------------------
+
+        # KITTI DATASET (social distancing)
+        # ------------------------------------------------------------------------------------------------------------
+        else:
+            self.dir_data = 'data/kitti/gt_activity'
+            self.dir_kk = os.path.join('data', 'kitti', 'calib')
+            self.THRESHOLD_PROB = 0.25  # Concordance for samples
+            self.THRESHOLD_DIST = 2  # Threshold to check distance of people
+            self.RADII = (0.3, 0.5, 1)  # expected radii of the o-space
+            self.PIFPAF_CONF = 0.3
+            self.SOCIAL_DISTANCE = True
+            # ---------------------------------------------------------------------------------------------------------
+
+        # Load model
+        device = torch.device('cpu')
+        if torch.cuda.is_available():
+            device = torch.device('cuda')
+        self.monoloco = Loco(
+            model=args.model,
+            mode=args.mode,
+            device=device,
+            n_dropout=args.n_dropout,
+            p_dropout=args.dropout)
+
+        self.all_pred = defaultdict(list)
+        self.all_gt = defaultdict(list)
+        assert args.dataset in ('collective', 'kitti')
+
+    def eval_collective(self):
+        """Parse Collective Activity Dataset and predict if people are talking or not"""
+
+        for seq in self.sequences:
+            images = glob.glob(os.path.join(self.dir_data, 'images',  seq + '*.jpg'))
+            initial_im = os.path.join(self.dir_data, 'images', seq + '_frame0001.jpg')
+            with open(initial_im, 'rb') as f:
+                image = Image.open(f).convert('RGB')
+                im_size = image.size
+                assert len(im_size) > 1, "image with frame0001 not available"
+
+            for im_path in images:
+
+                # Collect PifPaf files and calibration
+                basename = os.path.basename(im_path)
+                extension = '.predictions.json'
+                path_pif = os.path.join(self.dir_ann, basename + extension)
+                annotations = open_annotations(path_pif)
+                kk, _ = factory_for_gt(im_size)
+
+                # Collect corresponding gt files (ys_gt: 1 or 0)
+                boxes_gt, ys_gt = parse_gt_collective(self.dir_data, seq, path_pif)
+                # Run Monoloco
+                dic_out, boxes = self.run_monoloco(annotations, kk, im_size=im_size)
+
+                # Match and update stats
+                matches = get_iou_matches(boxes, boxes_gt, iou_min=0.3)
+
+                # Estimate activity
+                categories = [seq] * len(boxes_gt)  # for compatibility with KITTI evaluation
+                self.estimate_activity(dic_out, matches, ys_gt, categories=categories)
+
+        # Print Results
+            acc = ACCURACY_SCORE(self.all_gt[seq], self.all_pred[seq])
+            print(f"Accuracy of category {seq}: {100*acc:.2f}%")
+        cout_results(self.cnt, self.all_gt, self.all_pred, categories=self.sequences)
+
+    def eval_kitti(self):
+        """Parse KITTI Dataset and predict if people are talking or not"""
+        files = glob.glob(self.dir_data + '/*.txt')
+        # files = [self.dir_gt_kitti + '/001782.txt']
+        assert files, "Empty directory"
+
+        for file in files:
+
+            # Collect PifPaf files and calibration
+            basename, _ = os.path.splitext(os.path.basename(file))
+            path_calib = os.path.join(self.dir_kk, basename + '.txt')
+            annotations, kk, _ = factory_file(path_calib, self.dir_ann, basename)
+
+            # Collect corresponding gt files (ys_gt: 1 or 0)
+            path_gt = os.path.join(self.dir_data, basename + '.txt')
+            boxes_gt, ys_gt, difficulties = parse_gt_kitti(path_gt)
+
+            # Run Monoloco
+            dic_out, boxes = self.run_monoloco(annotations, kk, im_size=(1242, 374))
+
+            # Match and update stats
+            matches = get_iou_matches(boxes, boxes_gt, iou_min=0.3)
+
+            # Estimate activity
+            self.estimate_activity(dic_out, matches, ys_gt, categories=difficulties)
+
+            # Print Results
+        cout_results(self.cnt, self.all_gt, self.all_pred, categories=('easy', 'moderate', 'hard'))
+
+    def estimate_activity(self, dic_out, matches, ys_gt, categories):
+
+        # Calculate social interaction
+        angles = dic_out['angles']
+        dds = dic_out['dds_pred']
+        stds = dic_out['stds_ale']
+        xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']]
+
+        # Count gt statistics. (One element each gt)
+        for key in categories:
+            self.cnt['gt'][key] += 1
+            self.cnt['gt']['all'] += 1
+
+        for (idx, idx_gt) in matches:
+
+            # Select keys to update results for Collective or KITTI
+            keys = ('all', categories[idx_gt])
+
+            # Run social interactions rule
+            flag = social_interactions(idx, xz_centers, angles, dds,
+                                       stds=stds,
+                                       threshold_prob=self.THRESHOLD_PROB,
+                                       threshold_dist=self.THRESHOLD_DIST,
+                                       radii=self.RADII,
+                                       social_distance=self.SOCIAL_DISTANCE)
+            # Accumulate results
+            for key in keys:
+                self.all_pred[key].append(flag)
+                self.all_gt[key].append(ys_gt[idx_gt])
+                self.cnt['pred'][key] += 1
+
+    def run_monoloco(self, annotations, kk, im_size=None):
+
+        boxes, keypoints = preprocess_pifpaf(annotations, im_size, enlarge_boxes=True, min_conf=self.PIFPAF_CONF)
+        dic_out = self.monoloco.forward(keypoints, kk)
+        dic_out = self.monoloco.post_process(dic_out, boxes, keypoints, kk, dic_gt=None, reorder=False, verbose=False)
+
+        return dic_out, boxes
+
+
+def parse_gt_collective(dir_data, seq, path_pif):
+    """Parse both gt and binary label (1/0) for talking or not"""
+
+    path = os.path.join(dir_data, 'annotations', seq + '_annotations.txt')
+
+    with open(path, "r") as ff:
+        reader = csv.reader(ff, delimiter='\t')
+        dic_frames = defaultdict(lambda: defaultdict(list))
+        for line in reader:
+            box = convert_box(line[1:5])
+            cat = convert_category(line[5])
+            dic_frames[line[0]]['boxes'].append(box)
+            dic_frames[line[0]]['y'].append(cat)
+
+    frame = extract_frame_number(path_pif)
+    boxes_gt = dic_frames[frame]['boxes']
+    ys_gt = np.array(dic_frames[frame]['y'])
+    return boxes_gt, ys_gt
+
+
+def parse_gt_kitti(path_gt):
+    """Parse both gt and binary label (1/0) for talking or not"""
+    boxes_gt = []
+    ys = []
+    difficulties = []
+    with open(path_gt, "r") as f_gt:
+        for line_gt in f_gt:
+            line = line_gt.split()
+            box = [float(x) for x in line[4:8]]
+            boxes_gt.append(box)
+            y = int(line[-1])
+            assert y in (1, 0), "Expected to be binary (1/0)"
+            ys.append(y)
+            trunc = float(line[1])
+            occ = int(line[2])
+            difficulties.append(get_difficulty(box, trunc, occ))
+    return boxes_gt, ys, difficulties
+
+
+def cout_results(cnt, all_gt, all_pred, categories=()):
+
+    categories = list(categories)
+    categories.append('all')
+    print('-' * 80)
+
+    # Split by folders for collective activity
+    for key in categories:
+        acc = accuracy_score(all_gt[key], all_pred[key])
+        print("Accuracy of category {}: {:.2f}% , Recall: {:.2f}%, #: {}, Pred/Real positive: {:.1f}% / {:.1f}%"
+              .format(key,
+                      acc * 100,
+                      cnt['pred'][key] / cnt['gt'][key]*100,
+                      cnt['pred'][key],
+                      sum(all_pred[key]) / len(all_pred[key]) * 100,
+                      sum(all_gt[key]) / len(all_gt[key]) * 100
+                      )
+              )
+
+    # Final Accuracy
+    acc = accuracy_score(all_gt['all'], all_pred['all'])
+    recall = cnt['pred']['all'] / cnt['gt']['all'] * 100  # only predictions that match a ground-truth are included
+    print('-' * 80)
+    print(f"Final Accuracy: {acc * 100:.2f}      Final Recall:{recall:.2f}")
+    print('-' * 80)
+
+
+def convert_box(box_str):
+    """from string with left and center to standard """
+    box = [float(el) for el in box_str]  # x, y, w h
+    box[2] += box[0]
+    box[3] += box[1]
+    return box
+
+
+def convert_category(cat):
+    """Talking = 6"""
+    if cat == '6':
+        return 1
+    return 0
+
+
+def extract_frame_number(path):
+    """extract frame number from path"""
+    name = os.path.basename(path)
+    if name[11] == '0':
+        frame = name[12:15]
+    else:
+        frame = name[11:15]
+    return frame

monoloco/eval/eval_kitti.py

@@ -1,158 +1,209 @@
-"""Evaluate Monoloco code on KITTI dataset using ALE and ALP metrics with the following baselines:
+"""
-    - Mono3D
+Evaluate MonStereo code on KITTI dataset using ALE metric
-    - 3DOP
+"""
-    - MonoDepth
+
-    """
+# pylint: disable=attribute-defined-outside-init
 
 import os
 import math
 import logging
+import copy
 import datetime
 from collections import defaultdict
-from itertools import chain
 
-from tabulate import tabulate
+import numpy as np
+try:
+    import tabulate
+    TABULATE = copy.copy(tabulate.tabulate)
+except ImportError:
+    TABULATE = None
 
-from ..utils import get_iou_matches, get_task_error, get_pixel_error, check_conditions, get_category, split_training, \
+from ..utils import get_iou_matches, get_task_error, get_pixel_error, check_conditions, \
-    parse_ground_truth
+    get_difficulty, split_training, get_iou_matches_matrix, average, find_cluster
-from ..visuals import show_results, show_spread, show_task_error
+from ..prep import parse_ground_truth
+from ..visuals import show_results, show_spread, show_task_error, show_box_plot
 
 
 class EvalKitti:
 
     logging.basicConfig(level=logging.INFO)
     logger = logging.getLogger(__name__)
-    CLUSTERS = ('easy', 'moderate', 'hard', 'all', '6', '10', '15', '20', '25', '30', '40', '50', '>50')
+    CLUSTERS = ('easy', 'moderate', 'hard', 'all', '3', '5', '7', '9', '11', '13', '15', '17', '19', '21', '23', '25',
+                '27', '29', '31', '49')
     ALP_THRESHOLDS = ('<0.5m', '<1m', '<2m')
-    METHODS_MONO = ['m3d', 'monodepth', '3dop', 'monoloco']
+    OUR_METHODS = ['geometric', 'monoloco', 'monoloco_pp', 'pose', 'reid', 'monstereo']
-    METHODS_STEREO = ['ml_stereo', 'pose', 'reid']
+    METHODS_MONO = ['m3d', 'monopsr', 'smoke', 'monodis']
-    BASELINES = ['geometric', 'task_error', 'pixel_error']
+    METHODS_STEREO = ['3dop', 'psf', 'pseudo-lidar', 'e2e', 'oc-stereo']
+    BASELINES = ['task_error', 'pixel_error']
     HEADERS = ('method', '<0.5', '<1m', '<2m', 'easy', 'moderate', 'hard', 'all')
-    CATEGORIES = ('pedestrian',)
+    CATEGORIES = ('pedestrian',)  # extendable with person_sitting and/or cyclists
+    methods = OUR_METHODS + METHODS_MONO + METHODS_STEREO
 
-    def __init__(self, thresh_iou_monoloco=0.3, thresh_iou_base=0.3, thresh_conf_monoloco=0.3, thresh_conf_base=0.3,
+    # Set directories
-                 verbose=False, stereo=False):
+    main_dir = os.path.join('data', 'kitti')
-
+    dir_gt = os.path.join(main_dir, 'gt')
-        self.main_dir = os.path.join('data', 'kitti')
-        self.dir_gt = os.path.join(self.main_dir, 'gt')
-        self.methods = self.METHODS_MONO
-        self.stereo = stereo
-        if self.stereo:
-            self.methods.extend(self.METHODS_STEREO)
     path_train = os.path.join('splits', 'kitti_train.txt')
     path_val = os.path.join('splits', 'kitti_val.txt')
     dir_logs = os.path.join('data', 'logs')
-        assert dir_logs, "No directory to save final statistics"
+    assert os.path.exists(dir_logs), "No directory to save final statistics"
+    dir_fig = os.path.join('figures', 'results')
+
+    # Set thresholds to obtain comparable recalls
+    thresh_iou_monoloco = 0.3
+    thresh_iou_base = 0.3
+    thresh_conf_monoloco = 0.2
+    thresh_conf_base = 0.5
+
+    def __init__(self, args):
+        self.mode = args.mode
+        assert self.mode in ('mono', 'stereo'), "mode not recognized"
+        self.net = 'monstereo' if self.mode == 'stereo' else 'monoloco_pp'
+        self.verbose = args.verbose
+        self.save = args.save
+        self.show = args.show
 
         now = datetime.datetime.now()
         now_time = now.strftime("%Y%m%d-%H%M")[2:]
-        self.path_results = os.path.join(dir_logs, 'eval-' + now_time + '.json')
+        self.path_results = os.path.join(self.dir_logs, 'eval-' + now_time + '.json')
-        self.verbose = verbose
 
-        self.dic_thresh_iou = {method: (thresh_iou_monoloco if method[:8] == 'monoloco' else thresh_iou_base)
+        # Set thresholds for comparable recalls
+        self.dic_thresh_iou = {method: (self.thresh_iou_monoloco if method in self.OUR_METHODS
+                                        else self.thresh_iou_base)
                                for method in self.methods}
-        self.dic_thresh_conf = {method: (thresh_conf_monoloco if method[:8] == 'monoloco' else thresh_conf_base)
+        self.dic_thresh_conf = {method: (self.thresh_conf_monoloco if method in self.OUR_METHODS
+                                         else self.thresh_conf_base)
                                 for method in self.methods}
 
+        # Set thresholds to obtain comparable recall
+        self.dic_thresh_conf['monopsr'] += 0.4
+        self.dic_thresh_conf['e2e-pl'] = -100
+        self.dic_thresh_conf['oc-stereo'] = -100
+        self.dic_thresh_conf['smoke'] = -100
+        self.dic_thresh_conf['monodis'] = -100
+
         # Extract validation images for evaluation
         names_gt = tuple(os.listdir(self.dir_gt))
-        _, self.set_val = split_training(names_gt, path_train, path_val)
+        _, self.set_val = split_training(names_gt, self.path_train, self.path_val)
+
+        # self.set_val = ('002282.txt', )
 
         # Define variables to save statistics
-        self.dic_methods = None
+        self.dic_methods = self.errors = self.dic_stds = self.dic_stats = self.dic_cnt = self.cnt_gt = self.category \
-        self.errors = None
+            = None
-        self.dic_stds = None
+        self.cnt = 0
-        self.dic_stats = None
+
-        self.dic_cnt = None
+        # Filter methods with empty or non existent directory
-        self.cnt_gt = 0
+        filter_directories(self.main_dir, self.methods)
 
     def run(self):
         """Evaluate Monoloco performances on ALP and ALE metrics"""
 
-        for category in self.CATEGORIES:
+        for self.category in self.CATEGORIES:
-
             # Initialize variables
             self.errors = defaultdict(lambda: defaultdict(list))
-            self.dic_stds = defaultdict(lambda: defaultdict(list))
+            self.dic_stds = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
             self.dic_stats = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(float))))
             self.dic_cnt = defaultdict(int)
-            self.cnt_gt = 0
+            self.cnt_gt = defaultdict(int)
 
             # Iterate over each ground truth file in the training set
+            # self.set_val = ('000063.txt',)
             for name in self.set_val:
                 path_gt = os.path.join(self.dir_gt, name)
+                self.name = name
 
                 # Iterate over each line of the gt file and save box location and distances
-                out_gt = parse_ground_truth(path_gt, category)
+                out_gt = parse_ground_truth(path_gt, self.category)
                 methods_out = defaultdict(tuple)  # Save all methods for comparison
-                self.cnt_gt += len(out_gt[0])
+
+                # Count ground_truth:
+                boxes_gt, _, truncs_gt, occs_gt, _ = out_gt  # pylint: disable=unbalanced-tuple-unpacking
+                for idx, box in enumerate(boxes_gt):
+                    mode = get_difficulty(box, truncs_gt[idx], occs_gt[idx])
+                    self.cnt_gt[mode] += 1
+                    self.cnt_gt['all'] += 1
 
                 if out_gt[0]:
                     for method in self.methods:
                         # Extract annotations
                         dir_method = os.path.join(self.main_dir, method)
-                        assert os.path.exists(dir_method), "directory of the method %s does not exists" % method
                         path_method = os.path.join(dir_method, name)
-                        methods_out[method] = self._parse_txts(path_method, category, method=method)
+                        methods_out[method] = self._parse_txts(path_method, method=method)
 
                         # Compute the error with ground truth
                         self._estimate_error(out_gt, methods_out[method], method=method)
 
-                    # Iterate over all the files together to find a pool of common annotations
-                    self._compare_error(out_gt, methods_out)
-
             # Update statistics of errors and uncertainty
             for key in self.errors:
-                add_true_negatives(self.errors[key], self.cnt_gt)
+                add_true_negatives(self.errors[key], self.cnt_gt['all'])
-                for clst in self.CLUSTERS[:-2]:  # M3d and pifpaf does not have annotations above 40 meters
+                for clst in self.CLUSTERS[:-1]:
-                    get_statistics(self.dic_stats['test'][key][clst], self.errors[key][clst], self.dic_stds[clst], key)
+
+                    try:
+                        get_statistics(self.dic_stats['test'][key][clst],
+                                       self.errors[key][clst],
+                                       self.dic_stds[key][clst], key)
+                    except ZeroDivisionError:
+                        print('\n'+'-'*100 + '\n'+f'ERROR: method {key} at cluster {clst} is empty' + '\n'+'-'*100+'\n')
+                        raise
 
             # Show statistics
-            print('\n' + category.upper() + ':')
+            print('\n' + self.category.upper() + ':')
             self.show_statistics()
 
-    def printer(self, show, save):
+    def printer(self):
-        if save or show:
+        if self.save:
-            show_results(self.dic_stats, show, save, stereo=self.stereo)
+            os.makedirs(self.dir_fig, exist_ok=True)
-            show_spread(self.dic_stats, show, save)
+        if self.save or self.show:
-            show_task_error(show, save)
+            print('-' * 100)
+            show_results(self.dic_stats, self.CLUSTERS, self.net, self.dir_fig, show=self.show, save=self.save)
+            show_spread(self.dic_stats, self.CLUSTERS, self.net, self.dir_fig, show=self.show, save=self.save)
+            if self.net == 'monstereo':
+                show_box_plot(self.errors, self.CLUSTERS, self.dir_fig, show=self.show, save=self.save)
+            else:
+                show_task_error(self.dir_fig, show=self.show, save=self.save)
 
-    def _parse_txts(self, path, category, method):
+    def _parse_txts(self, path, method):
 
         boxes = []
         dds = []
-        stds_ale = []
+        cat = []
-        stds_epi = []
-        dds_geometric = []
-        output = (boxes, dds) if method != 'monoloco' else (boxes, dds, stds_ale, stds_epi, dds_geometric)
 
+        if method == 'psf':
+            path = os.path.splitext(path)[0] + '.png.txt'
+        if method in self.OUR_METHODS:
+            bis, epis = [], []
+            output = (boxes, dds, cat, bis, epis)
+        else:
+            output = (boxes, dds, cat)
         try:
             with open(path, "r") as ff:
                 for line_str in ff:
-                    line = line_str.split()
+                    if method == 'psf':
-                    if check_conditions(line, category, method=method, thresh=self.dic_thresh_conf[method]):
+                        line = line_str.split(", ")
-                        if method == 'monodepth':
-                            box = [float(x[:-1]) for x in line[0:4]]
-                            delta_h = (box[3] - box[1]) / 7
-                            delta_w = (box[2] - box[0]) / 3.5
-                            assert delta_h > 0 and delta_w > 0, "Bounding box <=0"
-                            box[0] -= delta_w
-                            box[1] -= delta_h
-                            box[2] += delta_w
-                            box[3] += delta_h
-                            dd = float(line[5][:-1])
-                        else:
                         box = [float(x) for x in line[4:8]]
+                        boxes.append(box)
                         loc = ([float(x) for x in line[11:14]])
                         dd = math.sqrt(loc[0] ** 2 + loc[1] ** 2 + loc[2] ** 2)
+                        dds.append(dd)
+                        cat.append('Pedestrian')
+                    else:
+                        line = line_str.split()
+                        if check_conditions(line,
+                                            category='pedestrian',
+                                            method=method,
+                                            thresh=self.dic_thresh_conf[method]):
+                            box = [float(x) for x in line[4:8]]
+                            box.append(float(line[15]))  # Add confidence
+                            loc = ([float(x) for x in line[11:14]])
+                            dd = math.sqrt(loc[0] ** 2 + loc[1] ** 2 + loc[2] ** 2)
+                            cat.append(line[0])
                             boxes.append(box)
                             dds.append(dd)
+                            if method in self.OUR_METHODS:
+                                bis.append(float(line[16]))
+                                epis.append(float(line[17]))
                             self.dic_cnt[method] += 1
-                        if method == 'monoloco':
+
-                            stds_ale.append(float(line[16]))
-                            stds_epi.append(float(line[17]))
-                            dds_geometric.append(float(line[18]))
-                            self.dic_cnt['geometric'] += 1
             return output
         except FileNotFoundError:
             return output
@@ -160,67 +211,39 @@ class EvalKitti:
     def _estimate_error(self, out_gt, out, method):
         """Estimate localization error"""
 
-        boxes_gt, _, dds_gt, zzs_gt, truncs_gt, occs_gt = out_gt
+        boxes_gt, ys, truncs_gt, occs_gt, _ = out_gt
-        if method == 'monoloco':
-            boxes, dds, stds_ale, stds_epi, dds_geometric = out
-        else:
-            boxes, dds = out
 
+        if method in self.OUR_METHODS:
+            boxes, dds, cat, bis, epis = out
+        else:
+            boxes, dds, cat = out
+
+        if method == 'psf':
+            matches = get_iou_matches_matrix(boxes, boxes_gt, self.dic_thresh_iou[method])
+        else:
             matches = get_iou_matches(boxes, boxes_gt, self.dic_thresh_iou[method])
 
         for (idx, idx_gt) in matches:
             # Update error if match is found
-            cat = get_category(boxes_gt[idx_gt], truncs_gt[idx_gt], occs_gt[idx_gt])
+            dd_gt = ys[idx_gt][3]
-            self.update_errors(dds[idx], dds_gt[idx_gt], cat, self.errors[method])
+            zz_gt = ys[idx_gt][2]
+            mode = get_difficulty(boxes_gt[idx_gt], truncs_gt[idx_gt], occs_gt[idx_gt])
 
+            if cat[idx].lower() in (self.category, 'pedestrian'):
+                self.update_errors(dds[idx], dd_gt, mode, self.errors[method])
                 if method == 'monoloco':
-                self.update_errors(dds_geometric[idx], dds_gt[idx_gt], cat, self.errors['geometric'])
+                    dd_task_error = dd_gt + (get_task_error(zz_gt))**2
-                self.update_uncertainty(stds_ale[idx], stds_epi[idx], dds[idx], dds_gt[idx_gt], cat)
+                    dd_pixel_error = dd_gt + get_pixel_error(zz_gt)
-                dd_task_error = dds_gt[idx_gt] + (get_task_error(dds_gt[idx_gt]))**2
+                    self.update_errors(dd_task_error, dd_gt, mode, self.errors['task_error'])
-                self.update_errors(dd_task_error, dds_gt[idx_gt], cat, self.errors['task_error'])
+                    self.update_errors(dd_pixel_error, dd_gt, mode, self.errors['pixel_error'])
-                dd_pixel_error = dds_gt[idx_gt] + get_pixel_error(zzs_gt[idx_gt])
+                if method in self.OUR_METHODS:
-                self.update_errors(dd_pixel_error, dds_gt[idx_gt], cat, self.errors['pixel_error'])
+                    epi = max(epis[idx], bis[idx])
-
+                    self.update_uncertainty(bis[idx], epi, dds[idx], dd_gt, mode, self.dic_stds[method])
-    def _compare_error(self, out_gt, methods_out):
-        """Compare the error for a pool of instances commonly matched by all methods"""
-        boxes_gt, _, dds_gt, zzs_gt, truncs_gt, occs_gt = out_gt
-
-        # Find IoU matches
-        matches = []
-        boxes_monoloco = methods_out['monoloco'][0]
-        matches_monoloco = get_iou_matches(boxes_monoloco, boxes_gt, self.dic_thresh_iou['monoloco'])
-
-        base_methods = [method for method in self.methods if method != 'monoloco']
-        for method in base_methods:
-            boxes = methods_out[method][0]
-            matches.append(get_iou_matches(boxes, boxes_gt, self.dic_thresh_iou[method]))
-
-        # Update error of commonly matched instances
-        for (idx, idx_gt) in matches_monoloco:
-            check, indices = extract_indices(idx_gt, *matches)
-            if check:
-                cat = get_category(boxes_gt[idx_gt], truncs_gt[idx_gt], occs_gt[idx_gt])
-                dd_gt = dds_gt[idx_gt]
-
-                for idx_indices, method in enumerate(base_methods):
-                    dd = methods_out[method][1][indices[idx_indices]]
-                    self.update_errors(dd, dd_gt, cat, self.errors[method + '_merged'])
-
-                dd_monoloco = methods_out['monoloco'][1][idx]
-                dd_geometric = methods_out['monoloco'][4][idx]
-                self.update_errors(dd_monoloco, dd_gt, cat, self.errors['monoloco_merged'])
-                self.update_errors(dd_geometric, dd_gt, cat, self.errors['geometric_merged'])
-                self.update_errors(dd_gt + get_task_error(dd_gt), dd_gt, cat, self.errors['task_error_merged'])
-                dd_pixel = dd_gt + get_pixel_error(zzs_gt[idx_gt])
-                self.update_errors(dd_pixel, dd_gt, cat, self.errors['pixel_error_merged'])
-
-                for key in self.methods:
-                    self.dic_cnt[key + '_merged'] += 1
 
     def update_errors(self, dd, dd_gt, cat, errors):
         """Compute and save errors between a single box and the gt box which match"""
         diff = abs(dd - dd_gt)
-        clst = find_cluster(dd_gt, self.CLUSTERS)
+        clst = find_cluster(dd_gt, self.CLUSTERS[4:])
         errors['all'].append(diff)
         errors[cat].append(diff)
         errors[clst].append(diff)
@@ -241,46 +264,49 @@ class EvalKitti:
         else:
             errors['<2m'].append(0)
 
-    def update_uncertainty(self, std_ale, std_epi, dd, dd_gt, cat):
+    def update_uncertainty(self, std_ale, std_epi, dd, dd_gt, mode, dic_stds):
 
-        clst = find_cluster(dd_gt, self.CLUSTERS)
+        clst = find_cluster(dd_gt, self.CLUSTERS[4:])
-        self.dic_stds['all']['ale'].append(std_ale)
+        dic_stds['all']['ale'].append(std_ale)
-        self.dic_stds[clst]['ale'].append(std_ale)
+        dic_stds[clst]['ale'].append(std_ale)
-        self.dic_stds[cat]['ale'].append(std_ale)
+        dic_stds[mode]['ale'].append(std_ale)
-        self.dic_stds['all']['epi'].append(std_epi)
+        dic_stds['all']['epi'].append(std_epi)
-        self.dic_stds[clst]['epi'].append(std_epi)
+        dic_stds[clst]['epi'].append(std_epi)
-        self.dic_stds[cat]['epi'].append(std_epi)
+        dic_stds[mode]['epi'].append(std_epi)
+        dic_stds['all']['epi_rel'].append(std_epi / dd)
+        dic_stds[clst]['epi_rel'].append(std_epi / dd)
+        dic_stds[mode]['epi_rel'].append(std_epi / dd)
 
         # Number of annotations inside the confidence interval
         std = std_epi if std_epi > 0 else std_ale  # consider aleatoric uncertainty if epistemic is not calculated
         if abs(dd - dd_gt) <= std:
-            self.dic_stds['all']['interval'].append(1)
+            dic_stds['all']['interval'].append(1)
-            self.dic_stds[clst]['interval'].append(1)
+            dic_stds[clst]['interval'].append(1)
-            self.dic_stds[cat]['interval'].append(1)
+            dic_stds[mode]['interval'].append(1)
         else:
-            self.dic_stds['all']['interval'].append(0)
+            dic_stds['all']['interval'].append(0)
-            self.dic_stds[clst]['interval'].append(0)
+            dic_stds[clst]['interval'].append(0)
-            self.dic_stds[cat]['interval'].append(0)
+            dic_stds[mode]['interval'].append(0)
 
         # Annotations at risk inside the confidence interval
         if dd_gt <= dd:
-            self.dic_stds['all']['at_risk'].append(1)
+            dic_stds['all']['at_risk'].append(1)
-            self.dic_stds[clst]['at_risk'].append(1)
+            dic_stds[clst]['at_risk'].append(1)
-            self.dic_stds[cat]['at_risk'].append(1)
+            dic_stds[mode]['at_risk'].append(1)
 
             if abs(dd - dd_gt) <= std_epi:
-                self.dic_stds['all']['at_risk-interval'].append(1)
+                dic_stds['all']['at_risk-interval'].append(1)
-                self.dic_stds[clst]['at_risk-interval'].append(1)
+                dic_stds[clst]['at_risk-interval'].append(1)
-                self.dic_stds[cat]['at_risk-interval'].append(1)
+                dic_stds[mode]['at_risk-interval'].append(1)
             else:
-                self.dic_stds['all']['at_risk-interval'].append(0)
+                dic_stds['all']['at_risk-interval'].append(0)
-                self.dic_stds[clst]['at_risk-interval'].append(0)
+                dic_stds[clst]['at_risk-interval'].append(0)
-                self.dic_stds[cat]['at_risk-interval'].append(0)
+                dic_stds[mode]['at_risk-interval'].append(0)
 
         else:
-            self.dic_stds['all']['at_risk'].append(0)
+            dic_stds['all']['at_risk'].append(0)
-            self.dic_stds[clst]['at_risk'].append(0)
+            dic_stds[clst]['at_risk'].append(0)
-            self.dic_stds[cat]['at_risk'].append(0)
+            dic_stds[mode]['at_risk'].append(0)
 
         # Precision of uncertainty
         eps = 1e-4
@@ -288,12 +314,12 @@ class EvalKitti:
         prec_1 = abs(dd - dd_gt) / (std_epi + eps)
 
         prec_2 = abs(std_epi - task_error)
-        self.dic_stds['all']['prec_1'].append(prec_1)
+        dic_stds['all']['prec_1'].append(prec_1)
-        self.dic_stds[clst]['prec_1'].append(prec_1)
+        dic_stds[clst]['prec_1'].append(prec_1)
-        self.dic_stds[cat]['prec_1'].append(prec_1)
+        dic_stds[mode]['prec_1'].append(prec_1)
-        self.dic_stds['all']['prec_2'].append(prec_2)
+        dic_stds['all']['prec_2'].append(prec_2)
-        self.dic_stds[clst]['prec_2'].append(prec_2)
+        dic_stds[clst]['prec_2'].append(prec_2)
-        self.dic_stds[cat]['prec_2'].append(prec_2)
+        dic_stds[mode]['prec_2'].append(prec_2)
 
     def show_statistics(self):
 
@@ -301,9 +327,21 @@ class EvalKitti:
         print('-'*90)
         self.summary_table(all_methods)
 
+        # Uncertainty
+        for net in ('monoloco_pp', 'monstereo'):
+            print(('-'*100))
+            print(net.upper())
+            for clst in ('easy', 'moderate', 'hard', 'all'):
+                print(" Annotations in clst {}: {:.0f}, Recall: {:.1f}. Precision: {:.2f}, Relative size is {:.1f} %"
+                      .format(clst,
+                              self.dic_stats['test'][net][clst]['cnt'],
+                              self.dic_stats['test'][net][clst]['interval']*100,
+                              self.dic_stats['test'][net][clst]['prec_1'],
+                              self.dic_stats['test'][net][clst]['epi_rel']*100))
+
         if self.verbose:
-            all_methods_merged = list(chain.from_iterable((method, method + '_merged') for method in all_methods))
+            for key in all_methods:
-            for key in all_methods_merged:
+                print(key.upper())
                 for clst in self.CLUSTERS[:4]:
                     print(" {} Average error in cluster {}: {:.2f} with a max error of {:.1f}, "
                           "for {} annotations"
@@ -311,22 +349,14 @@ class EvalKitti:
                                   self.dic_stats['test'][key][clst]['max'],
                                   self.dic_stats['test'][key][clst]['cnt']))
 
-                    if key == 'monoloco':
-                        print("% of annotation inside the confidence interval: {:.1f} %, "
-                              "of which {:.1f} % at higher risk"
-                              .format(self.dic_stats['test'][key][clst]['interval']*100,
-                                      self.dic_stats['test'][key][clst]['at_risk']*100))
-
                 for perc in self.ALP_THRESHOLDS:
                     print("{} Instances with error {}: {:.2f} %"
                           .format(key, perc, 100 * average(self.errors[key][perc])))
 
                 print("\nMatched annotations: {:.1f} %".format(self.errors[key]['matched']))
-                print(" Detected annotations : {}/{} ".format(self.dic_cnt[key], self.cnt_gt))
+                print(" Detected annotations : {}/{} ".format(self.dic_cnt[key], self.cnt_gt['all']))
                 print("-" * 100)
 
-            print("\n Annotations inside the confidence interval: {:.1f} %"
-                  .format(self.dic_stats['test']['monoloco']['all']['interval']))
             print("precision 1: {:.2f}".format(self.dic_stats['test']['monoloco']['all']['prec_1']))
             print("precision 2: {:.2f}".format(self.dic_stats['test']['monoloco']['all']['prec_2']))
 
@@ -337,26 +367,46 @@ class EvalKitti:
                 for perc in ['<0.5m', '<1m', '<2m']]
                for key in all_methods]
 
-        ale = [[str(self.dic_stats['test'][key + '_merged'][clst]['mean'])[:4] + ' (' +
+        ale = [[str(round(self.dic_stats['test'][key][clst]['mean'], 2))[:4] + ' [' +
-                str(self.dic_stats['test'][key][clst]['mean'])[:4] + ')'
+                str(round(self.dic_stats['test'][key][clst]['cnt'] / self.cnt_gt[clst] * 100))[:2] + '%]'
                 for clst in self.CLUSTERS[:4]]
                for key in all_methods]
 
         results = [[key] + alp[idx] + ale[idx] for idx, key in enumerate(all_methods)]
-        print(tabulate(results, headers=self.HEADERS))
+        print(TABULATE(results, headers=self.HEADERS))
         print('-' * 90 + '\n')
 
+    def stats_height(self):
+        heights = []
+        for name in self.set_val:
+            path_gt = os.path.join(self.dir_gt, name)
+            self.name = name
+            # Iterate over each line of the gt file and save box location and distances
+            out_gt = parse_ground_truth(path_gt, 'pedestrian')
+            for label in out_gt[1]:
+                heights.append(label[4])
+        tail1, tail2 = np.nanpercentile(np.array(heights), [5, 95])
+        print(average(heights))
+        print(len(heights))
+        print(tail1, tail2)
+
 
 def get_statistics(dic_stats, errors, dic_stds, key):
     """Update statistics of a cluster"""
 
+    try:
         dic_stats['mean'] = average(errors)
         dic_stats['max'] = max(errors)
         dic_stats['cnt'] = len(errors)
+    except ValueError:
+        dic_stats['mean'] = - 1
+        dic_stats['max'] = - 1
+        dic_stats['cnt'] = - 1
 
-    if key == 'monoloco':
+    if key in ('monoloco', 'monoloco_pp', 'monstereo'):
         dic_stats['std_ale'] = average(dic_stds['ale'])
         dic_stats['std_epi'] = average(dic_stds['epi'])
+        dic_stats['epi_rel'] = average(dic_stds['epi_rel'])
         dic_stats['interval'] = average(dic_stds['interval'])
         dic_stats['at_risk'] = average(dic_stds['at_risk'])
         dic_stats['prec_1'] = average(dic_stds['prec_1'])
@@ -375,16 +425,6 @@ def add_true_negatives(err, cnt_gt):
     err['matched'] = 100 * matched / cnt_gt
 
 
-def find_cluster(dd, clusters):
-    """Find the correct cluster. The first and the last one are not numeric"""
-
-    for clst in clusters[4: -1]:
-        if dd <= int(clst):
-            return clst
-
-    return clusters[-1]
-
-
 def extract_indices(idx_to_check, *args):
     """
     Look if a given index j_gt is present in all the other series of indices (_, j)
@@ -407,6 +447,12 @@ def extract_indices(idx_to_check, *args):
     return all(checks), indices
 
 
-def average(my_list):
+def filter_directories(main_dir, methods):
-    """calculate mean of a list"""
+    for method in methods:
-    return sum(my_list) / len(my_list)
+        dir_method = os.path.join(main_dir, method)
+        if not os.path.exists(dir_method):
+            methods.remove(method)
+            print(f"\nMethod {method}. No directory found. Skipping it..")
+        elif not os.listdir(dir_method):
+            methods.remove(method)
+            print(f"\nMethod {method}. Directory is empty. Skipping it..")

monoloco/eval/eval_variance.py

@@ -0,0 +1,218 @@
+# pylint: disable=too-many-statements
+
+"""Joints Analysis: Supplementary material of MonStereo"""
+
+import json
+import os
+from collections import defaultdict
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+from ..utils import find_cluster, average
+from ..visuals.figures import get_distances
+from ..prep.transforms import COCO_KEYPOINTS
+
+
+def joints_variance(joints, clusters, dic_ms):
+    # CLUSTERS = ('3', '5', '7', '9', '11', '13', '15', '17', '19', '21', '23', '25', '27', '29', '31', '49')
+    BF = 0.54 * 721
+    phase = 'train'
+    methods = ('pifpaf', 'mask')
+    dic_fin = {}
+
+    for method in methods:
+        dic_var = defaultdict(lambda: defaultdict(list))
+        dic_joints = defaultdict(list)
+        dic_avg = defaultdict(lambda: defaultdict(float))
+        path_joints = joints + '_' + method + '.json'
+
+        with open(path_joints, 'r') as f:
+            dic_jo = json.load(f)
+
+        for idx, keypoint in enumerate(dic_jo[phase]['kps']):
+            # if dic_jo[phase]['names'][idx] == '005856.txt' and dic_jo[phase]['Y'][idx][2] > 14:
+            #     aa = 4
+            assert len(keypoint) < 2
+            kps = np.array(keypoint[0])[:, :17]
+            kps_r = np.array(keypoint[0])[:, 17:]
+            disps = kps[0] - kps_r[0]
+            zz = dic_jo[phase]['Y'][idx][2]
+            disps_3 = get_variance(kps, kps_r, zz)
+            disps_8 = get_variance_conf(kps, kps_r, num=8)
+            disps_4 = get_variance_conf(kps, kps_r, num=4)
+            disp_gt = BF / zz
+            clst = find_cluster(zz, clusters)   # 4 = '3'    35 = '31'     42 = 2 = 'excl'
+            dic_var['std_d'][clst].append(disps.std())
+            errors = np.minimum(30, np.abs(zz - BF / disps))
+            dic_var['mean_dev'][clst].append(min(30, abs(zz - BF / np.median(disps))))
+            dic_var['mean_3'][clst].append(min(30, abs(zz - BF / disps_3.mean())))
+            dic_var['mean_8'][clst].append(min(30, abs(zz - BF / np.median(disps_8))))
+            dic_var['mean_4'][clst].append(min(30, abs(zz - BF / np.median(disps_4))))
+            arg_best = np.argmin(errors)
+            conf = np.mean((kps[2][arg_best], kps_r[2][arg_best]))
+            dic_var['mean_best'][clst].append(np.min(errors))
+            dic_var['conf_best'][clst].append(conf)
+            dic_var['conf'][clst].append(np.mean((np.mean(kps[2]), np.mean(kps_r[2]))))
+            # dic_var['std_z'][clst].append(zzs.std())
+            for ii, el in enumerate(disps):
+                if abs(el-disp_gt) < 1:
+                    dic_var['rep'][clst].append(1)
+                    dic_joints[str(ii)].append(1)
+                else:
+                    dic_var['rep'][clst].append(0)
+                    dic_joints[str(ii)].append(0)
+
+        for key in dic_var:
+            for clst in clusters[:-1]:  # 41 needs to be excluded (36 = '31')
+                dic_avg[key][clst] = average(dic_var[key][clst])
+        dic_fin[method] = dic_avg
+        for key in dic_joints:
+            dic_fin[method]['joints'][key] = average(dic_joints[key])
+        dic_fin['monstereo'] = {clst: dic_ms[clst]['mean'] for clst in clusters[:-1]}
+    variance_figures(dic_fin, clusters)
+
+
+def get_variance(kps, kps_r, zz):
+
+    thresh = 0.5 - zz / 100
+    disps_2 = []
+    disps = kps[0] - kps_r[0]
+    arg_disp = np.argsort(disps)[::-1]
+
+    for idx in arg_disp[1:]:
+        if kps[2][idx] > thresh and kps_r[2][idx] > thresh:
+            disps_2.append(disps[idx])
+        if len(disps_2) >= 3:
+            return np.array(disps_2)
+    return disps
+
+
+def get_variance_conf(kps, kps_r, num=8):
+
+    disps_conf = []
+    confs = (kps[2, :] + kps_r[2, :]) / 2
+    disps = kps[0] - kps_r[0]
+    arg_disp = np.argsort(confs)[::-1]
+
+    for idx in arg_disp[:num]:
+        disps_conf.append(disps[idx])
+    return np.array(disps_conf)
+
+
+def variance_figures(dic_fin, clusters):
+    """Predicted confidence intervals and task error as a function of ground-truth distance"""
+
+    dir_out = 'docs'
+    x_min = 3
+    x_max = 43
+    y_min = 0
+    y_max = 1
+
+    plt.figure(0)
+    plt.xlabel("Ground-truth distance [m]")
+    plt.title("Repeatability by distance")
+    plt.xlim(x_min, x_max)
+    plt.ylim(y_min, y_max)
+    plt.grid(linewidth=0.2)
+
+    xxs = get_distances(clusters)
+    yys_p = [el for _, el in dic_fin['pifpaf']['rep'].items()]
+    yys_m = [el for _, el in dic_fin['mask']['rep'].items()]
+    plt.plot(xxs, yys_p, marker='s', label="PifPaf")
+    plt.plot(xxs, yys_m, marker='o', label="Mask R-CNN")
+    plt.tight_layout()
+    plt.legend()
+    path_fig = os.path.join(dir_out, 'repeatability.png')
+    plt.savefig(path_fig)
+    print("Figure of repeatability saved in {}".format(path_fig))
+
+    plt.figure(1)
+    plt.xlabel("Ground-truth distance [m]")
+    plt.ylabel("[m]")
+    plt.title("Depth error")
+    plt.grid(linewidth=0.2)
+    y_min = 0
+    y_max = 2.7
+    plt.ylim(y_min, y_max)
+    yys_p = [el for _, el in dic_fin['pifpaf']['mean_dev'].items()]
+    # yys_m = [el for _, el in dic_fin['mask']['mean_dev'].items()]
+    yys_p_3 = [el for _, el in dic_fin['pifpaf']['mean_3'].items()]
+    yys_p_8 = [el for _, el in dic_fin['pifpaf']['mean_8'].items()]
+    yys_p_4 = [el for _, el in dic_fin['pifpaf']['mean_4'].items()]
+    # yys_m_3 = [el for _, el in dic_fin['mask']['mean_3'].items()]
+    yys_ms = [el for _, el in dic_fin['monstereo'].items()]
+    yys_p_best = [el for _, el in dic_fin['pifpaf']['mean_best'].items()]
+    plt.plot(xxs, yys_p_4, marker='o', linestyle=':', label="PifPaf (highest 4)")
+    plt.plot(xxs, yys_p, marker='+', label="PifPaf (median)")
+    # plt.plot(xxs, yys_m, marker='o', label="Mask R-CNN (median")
+    plt.plot(xxs, yys_p_3, marker='s', linestyle='--', label="PifPaf (closest 3)")
+    plt.plot(xxs, yys_p_8, marker='*', linestyle=':', label="PifPaf (highest 8)")
+    plt.plot(xxs, yys_ms, marker='^', label="MonStereo")
+    plt.plot(xxs, yys_p_best, marker='o', label="PifPaf (best)")
+    # plt.plot(xxs, yys_m_3, marker='o', color='r', label="Mask R-CNN (closest 3)")
+    # plt.plot(xxs, yys_mon, marker='o', color='b', label="Our MonStereo")
+
+    plt.legend()
+    plt.tight_layout()
+    path_fig = os.path.join(dir_out, 'mean_deviation.png')
+    plt.savefig(path_fig)
+    print("Figure of mean deviation saved in {}".format(path_fig))
+
+    plt.figure(2)
+    plt.xlabel("Ground-truth distance [m]")
+    plt.ylabel("Pixels")
+    plt.title("Standard deviation of joints disparity")
+    yys_p = [el for _, el in dic_fin['pifpaf']['std_d'].items()]
+    yys_m = [el for _, el in dic_fin['mask']['std_d'].items()]
+    # yys_p_z = [el for _, el in dic_fin['pifpaf']['std_z'].items()]
+    # yys_m_z = [el for _, el in dic_fin['mask']['std_z'].items()]
+    plt.plot(xxs, yys_p, marker='s', label="PifPaf")
+    plt.plot(xxs, yys_m, marker='o', label="Mask R-CNN")
+    # plt.plot(xxs, yys_p_z, marker='s', color='b', label="PifPaf (meters)")
+    # plt.plot(xxs, yys_m_z, marker='o', color='r', label="Mask R-CNN (meters)")
+
+    plt.grid(linewidth=0.2)
+    plt.legend()
+    path_fig = os.path.join(dir_out, 'std_joints.png')
+    plt.savefig(path_fig)
+    print("Figure of standard deviation of joints by distance in {}".format(path_fig))
+
+    plt.figure(3)
+    # plt.style.use('ggplot')
+    width = 0.35
+    xxs = np.arange(len(COCO_KEYPOINTS))
+    yys_p = [el for _, el in dic_fin['pifpaf']['joints'].items()]
+    yys_m = [el for _, el in dic_fin['mask']['joints'].items()]
+    plt.bar(xxs, yys_p, width, color='C0', label='Pifpaf')
+    plt.bar(xxs + width, yys_m, width, color='C1', label='Mask R-CNN')
+    plt.ylim(0, 1)
+
+    plt.xlabel("Keypoints")
+    plt.title("Repeatability by keypoint type")
+
+    plt.xticks(xxs + width / 2, xxs)
+    plt.legend(loc='best')
+    path_fig = os.path.join(dir_out, 'repeatability_2.png')
+    plt.savefig(path_fig)
+    plt.close('all')
+    print("Figure of standard deviation of joints by keypointd in {}".format(path_fig))
+
+    plt.figure(4)
+    plt.xlabel("Ground-truth distance [m]")
+    plt.ylabel("Confidence")
+    plt.grid(linewidth=0.2)
+    xxs = get_distances(clusters)
+    yys_p_conf = [el for _, el in dic_fin['pifpaf']['conf'].items()]
+    yys_p_conf_best = [el for _, el in dic_fin['pifpaf']['conf_best'].items()]
+    yys_m_conf = [el for _, el in dic_fin['mask']['conf'].items()]
+    yys_m_conf_best = [el for _, el in dic_fin['mask']['conf_best'].items()]
+    plt.plot(xxs, yys_p_conf_best, marker='s', color='lightblue', label="PifPaf (best)")
+    plt.plot(xxs, yys_p_conf, marker='s', color='b', label="PifPaf (mean)")
+    plt.plot(xxs, yys_m_conf_best, marker='^', color='darkorange', label="Mask (best)")
+    plt.plot(xxs, yys_m_conf, marker='o', color='r', label="Mask R-CNN (mean)")
+    plt.legend()
+    plt.tight_layout()
+    path_fig = os.path.join(dir_out, 'confidence.png')
+    plt.savefig(path_fig)
+    print("Figure of confidence saved in {}".format(path_fig))

monoloco/eval/generate_kitti.py

@@ -1,221 +1,277 @@
 
-"""Run monoloco over all the pifpaf joints of KITTI images
+# pylint: disable=too-many-branches
-and extract and save the annotations in txt files"""
 
+"""
+Run MonoLoco/MonStereo and converts annotations into KITTI format
+"""
 
 import os
-import glob
+import math
-import shutil
 from collections import defaultdict
 
-import numpy as np
 import torch
 
-from ..network import MonoLoco
+from ..network import Loco
 from ..network.process import preprocess_pifpaf
-from ..eval.geom_baseline import compute_distance
+from .geom_baseline import geometric_coordinates
-from ..utils import get_keypoints, pixel_to_camera, xyz_from_distance, get_calibration, open_annotations, split_training
+from ..utils import get_keypoints, pixel_to_camera, factory_basename, make_new_directory, get_category, \
+    xyz_from_distance, read_and_rewrite
 from .stereo_baselines import baselines_association
-from .reid_baseline import ReID, get_reid_features
+from ..prep import factory_file
+from .reid_baseline import get_reid_features, ReID
 
 
 class GenerateKitti:
 
-    def __init__(self, model, dir_ann, p_dropout=0.2, n_dropout=0, stereo=True):
+    dir_gt = os.path.join('data', 'kitti', 'gt')
+    dir_gt_new = os.path.join('data', 'kitti', 'gt_new')
+    dir_kk = os.path.join('data', 'kitti', 'calib')
+    dir_byc = '/data/lorenzo-data/kitti/object_detection/left'
+    monoloco_checkpoint = 'data/models/monoloco-190717-0952.pkl'
+    baselines = {'mono': [], 'stereo': []}
 
-        # Load monoloco
+    def __init__(self, args):
+
+        # Load Network
+        assert args.mode in ('mono', 'stereo'), "mode not recognized"
+        self.mode = args.mode
+        self.net = 'monstereo' if args.mode == 'stereo' else 'monoloco_pp'
         use_cuda = torch.cuda.is_available()
         device = torch.device("cuda" if use_cuda else "cpu")
-        self.monoloco = MonoLoco(model=model, device=device, n_dropout=n_dropout, p_dropout=p_dropout)
+        self.model = Loco(
-        self.dir_ann = dir_ann
+            model=args.model,
+            mode=args.mode,
+            device=device,
+            n_dropout=args.n_dropout,
+            p_dropout=args.dropout,
+            linear_size=args.hidden_size
+            )
 
         # Extract list of pifpaf files in validation images
-        dir_gt = os.path.join('data', 'kitti', 'gt')
+        self.dir_ann = args.dir_ann
-        self.set_basename = factory_basename(dir_ann, dir_gt)
+        self.generate_official = args.generate_official
-        self.dir_kk = os.path.join('data', 'kitti', 'calib')
+        assert os.listdir(self.dir_ann), "Annotation directory is empty"
+        self.set_basename = factory_basename(args.dir_ann, self.dir_gt)
 
-        # Calculate stereo baselines
+        # For quick testing
-        self.stereo = stereo
+        # ------------------------------------------------------------------------------------------------------------
-        if stereo:
+        # self.set_basename = ('001782',)
-            self.baselines = ['ml_stereo', 'pose', 'reid']
+        # self.set_basename = ('002282',)
+        # ------------------------------------------------------------------------------------------------------------
+
+        # Add monocular and stereo baselines (they require monoloco as backbone)
+        if args.baselines:
+            # Load MonoLoco
+            self.baselines['mono'] = ['monoloco', 'geometric']
+            self.monoloco = Loco(
+                model=self.monoloco_checkpoint,
+                mode='mono',
+                net='monoloco',
+                device=device,
+                n_dropout=args.n_dropout,
+                p_dropout=args.dropout,
+                linear_size=256
+            )
+            # Stereo baselines
+            if args.mode == 'stereo':
+                self.baselines['stereo'] = ['pose', 'reid']
                 self.cnt_disparity = defaultdict(int)
                 self.cnt_no_stereo = 0
+                self.dir_images = os.path.join('data', 'kitti', 'images')
+                self.dir_images_r = os.path.join('data', 'kitti', 'images_r')
 
                 # ReID Baseline
                 weights_path = 'data/models/reid_model_market.pkl'
                 self.reid_net = ReID(weights_path=weights_path, device=device, num_classes=751, height=256, width=128)
-            self.dir_images = os.path.join('data', 'kitti', 'images')
-            self.dir_images_r = os.path.join('data', 'kitti', 'images_r')
 
     def run(self):
         """Run Monoloco and save txt files for KITTI evaluation"""
 
         cnt_ann = cnt_file = cnt_no_file = 0
-        dir_out = {"monoloco": os.path.join('data', 'kitti', 'monoloco')}
-        make_new_directory(dir_out["monoloco"])
-        print("\nCreated empty output directory for txt files")
 
-        if self.stereo:
+        # Prepare empty folder
-            for key in self.baselines:
+        di = os.path.join('data', 'kitti', self.net)
-                dir_out[key] = os.path.join('data', 'kitti', key)
+        make_new_directory(di)
-                make_new_directory(dir_out[key])
+        dir_out = {self.net: di}
-                print("Created empty output directory for {}".format(key))
-            print("\n")
 
-        # Run monoloco over the list of images
+        for _, names in self.baselines.items():
+            for name in names:
+                di = os.path.join('data', 'kitti', name)
+                make_new_directory(di)
+                dir_out[name] = di
+
+        # Run the model
         for basename in self.set_basename:
             path_calib = os.path.join(self.dir_kk, basename + '.txt')
             annotations, kk, tt = factory_file(path_calib, self.dir_ann, basename)
             boxes, keypoints = preprocess_pifpaf(annotations, im_size=(1242, 374))
-            assert keypoints, "all pifpaf files should have at least one annotation"
+            cat = get_category(keypoints, os.path.join(self.dir_byc, basename + '.json'))
+            if keypoints:
+                annotations_r, _, _ = factory_file(path_calib, self.dir_ann, basename, ann_type='right')
+                _, keypoints_r = preprocess_pifpaf(annotations_r, im_size=(1242, 374))
+
+                if self.net == 'monstereo':
+                    dic_out = self.model.forward(keypoints, kk, keypoints_r=keypoints_r)
+                elif self.net == 'monoloco_pp':
+                    dic_out = self.model.forward(keypoints, kk)
+
+                all_outputs = {self.net: [dic_out['xyzd'], dic_out['bi'], dic_out['epi'],
+                                          dic_out['yaw'], dic_out['h'], dic_out['w'], dic_out['l']]}
+                zzs = [float(el[2]) for el in dic_out['xyzd']]
+
+                # Save txt files
+                params = [kk, tt]
+                path_txt = os.path.join(dir_out[self.net], basename + '.txt')
+                save_txts(path_txt, boxes, all_outputs[self.net], params, net=self.net, cat=cat)
                 cnt_ann += len(boxes)
                 cnt_file += 1
 
-            # Run the network and the geometric baseline
+                # MONO  (+ STEREO BASELINES)
-            outputs, varss = self.monoloco.forward(keypoints, kk)
+                if self.baselines['mono']:
-            dds_geom = eval_geometric(keypoints, kk, average_y=0.48)
+                    # MONOLOCO
+                    dic_out = self.monoloco.forward(keypoints, kk)
+                    zzs_geom, xy_centers = geometric_coordinates(keypoints, kk, average_y=0.48)
+                    all_outputs['monoloco'] = [dic_out['d'], dic_out['bi'], dic_out['epi']] + [zzs_geom, xy_centers]
+                    all_outputs['geometric'] = all_outputs['monoloco']
 
-            # Save the file
+                    # monocular baselines
-            uv_centers = get_keypoints(keypoints, mode='bottom')  # Kitti uses the bottom center to calculate depth
+                    for key in self.baselines['mono']:
-            xy_centers = pixel_to_camera(uv_centers, kk, 1)
+                        path_txt = {key: os.path.join(dir_out[key], basename + '.txt')}
-            outputs = outputs.detach().cpu()
+                        save_txts(path_txt[key], boxes, all_outputs[key], params, net=key, cat=cat)
-            zzs = xyz_from_distance(outputs[:, 0:1], xy_centers)[:, 2].tolist()
 
-            all_outputs = [outputs.detach().cpu(), varss.detach().cpu(), dds_geom, zzs]
+                    # stereo baselines
-            all_inputs = [boxes, xy_centers]
+                    if self.baselines['stereo']:
-            all_params = [kk, tt]
+                        all_inputs = {}
-            path_txt = {'monoloco': os.path.join(dir_out['monoloco'], basename + '.txt')}
+                        dic_xyz = self._run_stereo_baselines(basename, boxes, keypoints, zzs, path_calib)
-            save_txts(path_txt['monoloco'], all_inputs, all_outputs, all_params)
+                        for key in dic_xyz:
+                            all_outputs[key] = all_outputs['monoloco'].copy()
+                            all_outputs[key][0] = dic_xyz[key]
+                            all_inputs[key] = boxes
 
-            # Correct using stereo disparity and save in different folder
-            if self.stereo:
-                zzs = self._run_stereo_baselines(basename, boxes, keypoints, zzs, path_calib)
-                for key in zzs:
                             path_txt[key] = os.path.join(dir_out[key], basename + '.txt')
-                    save_txts(path_txt[key], all_inputs, zzs[key], all_params, mode='baseline')
+                            save_txts(path_txt[key], all_inputs[key], all_outputs[key], params,
+                                      net='baseline',
+                                      cat=cat)
 
         print("\nSaved in {} txt {} annotations. Not found {} images".format(cnt_file, cnt_ann, cnt_no_file))
 
-        if self.stereo:
+        if self.baselines[self.mode] and self.net == 'monstereo':
             print("STEREO:")
-            for key in self.baselines:
+            for key in self.baselines['stereo']:
                 print("Annotations corrected using {} baseline: {:.1f}%".format(
                     key, self.cnt_disparity[key] / cnt_ann * 100))
-            print("Maximum possible stereo associations: {:.1f}%".format(self.cnt_disparity['max'] / cnt_ann * 100))
             print("Not found {}/{} stereo files".format(self.cnt_no_stereo, cnt_file))
 
+        if self.generate_official:
+            create_empty_files(dir_out, self.net)  # Create empty files for official evaluation
+
     def _run_stereo_baselines(self, basename, boxes, keypoints, zzs, path_calib):
 
-        annotations_r, _, _ = factory_file(path_calib, self.dir_ann, basename, mode='right')
+        annotations_r, _, _ = factory_file(path_calib, self.dir_ann, basename, ann_type='right')
         boxes_r, keypoints_r = preprocess_pifpaf(annotations_r, im_size=(1242, 374))
+        _, kk, _ = factory_file(path_calib, self.dir_ann, basename)
+
+        uv_centers = get_keypoints(keypoints, mode='bottom')  # Kitti uses the bottom center to calculate depth
+        xy_centers = pixel_to_camera(uv_centers, kk, 1)
 
         # Stereo baselines
         if keypoints_r:
             path_image = os.path.join(self.dir_images, basename + '.png')
             path_image_r = os.path.join(self.dir_images_r, basename + '.png')
             reid_features = get_reid_features(self.reid_net, boxes, boxes_r, path_image, path_image_r)
-            zzs, cnt = baselines_association(self.baselines, zzs, keypoints, keypoints_r, reid_features)
+            dic_zzs, cnt = baselines_association(self.baselines['stereo'], zzs, keypoints, keypoints_r, reid_features)
 
             for key in cnt:
                 self.cnt_disparity[key] += cnt[key]
 
         else:
             self.cnt_no_stereo += 1
-            zzs = {key: zzs for key in self.baselines}
+            dic_zzs = {key: zzs for key in self.baselines['stereo']}
-        return zzs
+
+        # Combine the stereo zz with x, y from 2D detection (no MonoLoco involved)
+        dic_xyz = defaultdict(list)
+        for key in dic_zzs:
+            for idx, zz_base in enumerate(dic_zzs[key]):
+                xx = float(xy_centers[idx][0]) * zz_base
+                yy = float(xy_centers[idx][1]) * zz_base
+                dic_xyz[key].append([xx, yy, zz_base])
+
+        return dic_xyz
 
 
-def save_txts(path_txt, all_inputs, all_outputs, all_params, mode='monoloco'):
+def save_txts(path_txt, all_inputs, all_outputs, all_params, net='monoloco', cat=None):
 
-    assert mode in ('monoloco', 'baseline')
+    assert net in ('monoloco', 'monstereo', 'geometric', 'baseline', 'monoloco_pp')
-    if mode == 'monoloco':
+
-        outputs, varss, dds_geom, zzs = all_outputs[:]
+    if net in ('monstereo', 'monoloco_pp'):
+        xyzd, bis, epis, yaws, hs, ws, ls = all_outputs[:]
+        xyz = xyzd[:, 0:3]
+        tt = [0, 0, 0]
+    elif net in ('monoloco', 'geometric'):
+        tt = [0, 0, 0]
+        dds, bis, epis, zzs_geom, xy_centers = all_outputs[:]
+        xyz = xyz_from_distance(dds, xy_centers)
     else:
-        zzs = all_outputs
+        _, tt = all_params[:]
-    uv_boxes, xy_centers = all_inputs[:]
+        xyz, bis, epis, zzs_geom, xy_centers = all_outputs[:]
-    kk, tt = all_params[:]
+    uv_boxes = all_inputs[:]
+    assert len(uv_boxes) == len(list(xyz)), "Number of inputs different from number of outputs"
 
     with open(path_txt, "w+") as ff:
-        for idx, zz_base in enumerate(zzs):
+        for idx, uv_box in enumerate(uv_boxes):
+
+            xx = float(xyz[idx][0]) - tt[0]
+            yy = float(xyz[idx][1]) - tt[1]
+            zz = float(xyz[idx][2]) - tt[2]
+
+            if net == 'geometric':
+                zz = zzs_geom[idx]
 
-            xx = float(xy_centers[idx][0]) * zzs[idx] + tt[0]
-            yy = float(xy_centers[idx][1]) * zzs[idx] + tt[1]
-            zz = zz_base + tt[2]
             cam_0 = [xx, yy, zz]
-            output_list = [0.]*3 + uv_boxes[idx][:-1] + [0.]*3 + cam_0 + [0.] + uv_boxes[idx][-1:]  # kitti format
+            bi = float(bis[idx])
-            ff.write("%s " % 'pedestrian')
+            epi = float(epis[idx])
+            if net in ('monstereo', 'monoloco_pp'):
+                alpha, ry = float(yaws[0][idx]), float(yaws[1][idx])
+                hwl = [float(hs[idx]), float(ws[idx]), float(ls[idx])]
+                # scale to obtain (approximately) same recall at evaluation
+                conf_scale = 0.035 if net == 'monoloco_pp' else 0.033
+            else:
+                alpha, ry, hwl = -10., -10., [0, 0, 0]
+                conf_scale = 0.05
+            conf = conf_scale * (uv_box[-1]) / (bi / math.sqrt(xx ** 2 + yy ** 2 + zz ** 2))
+
+            output_list = [alpha] + uv_box[:-1] + hwl + cam_0 + [ry, conf, bi, epi]
+            category = cat[idx]
+            if category < 0.1:
+                ff.write("%s " % 'Pedestrian')
+            else:
+                ff.write("%s " % 'Cyclist')
+
+            ff.write("%i %i " % (-1, -1))
             for el in output_list:
                 ff.write("%f " % el)
-
-            # add additional uncertainty information
-            if mode == 'monoloco':
-                ff.write("%f " % float(outputs[idx][1]))
-                ff.write("%f " % float(varss[idx]))
-                ff.write("%f " % dds_geom[idx])
             ff.write("\n")
 
 
-def factory_file(path_calib, dir_ann, basename, mode='left'):
+def create_empty_files(dir_out, net):
-    """Choose the annotation and the calibration files. Stereo option with ite = 1"""
+    """Create empty txt files to run official kitti metrics on MonStereo and all other methods"""
 
-    assert mode in ('left', 'right')
+    methods = ['pseudo-lidar', 'monopsr', '3dop', 'm3d', 'oc-stereo', 'e2e', 'monodis', 'smoke']
-    p_left, p_right = get_calibration(path_calib)
+    dirs = [os.path.join('data', 'kitti', method) for method in methods]
+    dirs_orig = [os.path.join('data', 'kitti', method + '-orig') for method in methods]
 
-    if mode == 'left':
+    for di, di_orig in zip(dirs, dirs_orig):
-        kk, tt = p_left[:]
+        make_new_directory(di)
-        path_ann = os.path.join(dir_ann, basename + '.png.pifpaf.json')
 
-    else:
+        for i in range(7481):
-        kk, tt = p_right[:]
+            name = "0" * (6 - len(str(i))) + str(i) + '.txt'
-        path_ann = os.path.join(dir_ann + '_right', basename + '.png.pifpaf.json')
+            path_orig = os.path.join(di_orig, name)
+            path = os.path.join(di, name)
 
-    annotations = open_annotations(path_ann)
+            # If the file exits, rewrite in new folder, otherwise create empty file
+            read_and_rewrite(path_orig, path)
 
-    return annotations, kk, tt
+    for i in range(7481):
-
+        name = "0" * (6 - len(str(i))) + str(i) + '.txt'
-
+        with open(os.path.join(dir_out[net], name), "a+"):
-def eval_geometric(keypoints, kk, average_y=0.48):
+            pass
-    """ Evaluate geometric distance"""
-
-    dds_geom = []
-
-    uv_centers = get_keypoints(keypoints, mode='center')
-    uv_shoulders = get_keypoints(keypoints, mode='shoulder')
-    uv_hips = get_keypoints(keypoints, mode='hip')
-
-    xy_centers = pixel_to_camera(uv_centers, kk, 1)
-    xy_shoulders = pixel_to_camera(uv_shoulders, kk, 1)
-    xy_hips = pixel_to_camera(uv_hips, kk, 1)
-
-    for idx, xy_center in enumerate(xy_centers):
-        zz = compute_distance(xy_shoulders[idx], xy_hips[idx], average_y)
-        xyz_center = np.array([xy_center[0], xy_center[1], zz])
-        dd_geom = float(np.linalg.norm(xyz_center))
-        dds_geom.append(dd_geom)
-
-    return dds_geom
-
-
-def make_new_directory(dir_out):
-    """Remove the output directory if already exists (avoid residual txt files)"""
-    if os.path.exists(dir_out):
-        shutil.rmtree(dir_out)
-    os.makedirs(dir_out)
-
-
-def factory_basename(dir_ann, dir_gt):
-    """ Return all the basenames in the annotations folder corresponding to validation images"""
-
-    # Extract ground truth validation images
-    names_gt = tuple(os.listdir(dir_gt))
-    path_train = os.path.join('splits', 'kitti_train.txt')
-    path_val = os.path.join('splits', 'kitti_val.txt')
-    _, set_val_gt = split_training(names_gt, path_train, path_val)
-    set_val_gt = {os.path.basename(x).split('.')[0] for x in set_val_gt}
-
-    # Extract pifpaf files corresponding to validation images
-    list_ann = glob.glob(os.path.join(dir_ann, '*.json'))
-    set_basename = {os.path.basename(x).split('.')[0] for x in list_ann}
-    set_val = set_basename.intersection(set_val_gt)
-    assert set_val, " Missing json annotations file to create txt files for KITTI datasets"
-    return set_val

monoloco/eval/geom_baseline.py

@@ -1,17 +1,34 @@
 
 import json
-import logging
 import math
 from collections import defaultdict
 
 import numpy as np
 
-from ..utils import pixel_to_camera, get_keypoints
+from monoloco.utils import pixel_to_camera, get_keypoints
 
 AVERAGE_Y = 0.48
 CLUSTERS = ['10', '20', '30', 'all']
 
 
+def geometric_coordinates(keypoints, kk, average_y=0.48):
+    """ Evaluate geometric depths for a set of keypoints"""
+
+    zzs_geom = []
+    uv_shoulders = get_keypoints(keypoints, mode='shoulder')
+    uv_hips = get_keypoints(keypoints, mode='hip')
+    uv_centers = get_keypoints(keypoints, mode='center')
+
+    xy_shoulders = pixel_to_camera(uv_shoulders, kk, 1)
+    xy_hips = pixel_to_camera(uv_hips, kk, 1)
+    xy_centers = pixel_to_camera(uv_centers, kk, 1)
+
+    for idx, xy_shoulder in enumerate(xy_shoulders):
+        zz = compute_depth(xy_shoulder, xy_hips[idx], average_y)
+        zzs_geom.append(zz)
+    return zzs_geom, xy_centers
+
+
 def geometric_baseline(joints):
     """
     List of json files --> 2 lists with mean and std for each segment and the total count of instances
@@ -28,8 +45,6 @@ def geometric_baseline(joints):
     'right_ankle']
 
     """
-    logging.basicConfig(level=logging.INFO)
-    logger = logging.getLogger(__name__)
     cnt_tot = 0
     dic_dist = defaultdict(lambda: defaultdict(list))
 
@@ -48,13 +63,13 @@ def geometric_baseline(joints):
     errors = calculate_error(dic_dist['error'])
 
     # Show results
-    logger.info("Computed distance of {} annotations".format(cnt_tot))
+    print("Computed distance of {} annotations".format(cnt_tot))
     for key in dic_h_means:
-        logger.info("Average height of segment {} is {:.2f} with a std of {:.2f}".
+        print("Average height of segment {} is {:.2f} with a std of {:.2f}".
               format(key, dic_h_means[key], dic_h_stds[key]))
     for clst in CLUSTERS:
-        logger.info("Average error over the val set for clst {}: {:.2f}".format(clst, errors[clst]))
+        print("Average error over the val set for clst {}: {:.2f}".format(clst, errors[clst]))
-    logger.info("Joints used: {}".format(joints))
+    print("Joints used: {}".format(joints))
 
 
 def update_distances(dic_fin, dic_dist, phase, average_y):
@@ -78,9 +93,9 @@ def update_distances(dic_fin, dic_dist, phase, average_y):
         dy_met = abs(float((dic_xyz['hip'][0][1] - dic_xyz['shoulder'][0][1])))
 
         # Estimate distance for a single annotation
-        z_met_real = compute_distance(dic_xyz_norm['shoulder'][0], dic_xyz_norm['hip'][0], average_y,
+        z_met_real = compute_depth(dic_xyz_norm['shoulder'][0], dic_xyz_norm['hip'][0], average_y,
                                    mode='real', dy_met=dy_met)
-        z_met_approx = compute_distance(dic_xyz_norm['shoulder'][0], dic_xyz_norm['hip'][0], average_y, mode='average')
+        z_met_approx = compute_depth(dic_xyz_norm['shoulder'][0], dic_xyz_norm['hip'][0], average_y, mode='average')
 
         # Compute distance with respect to the center of the 3D bounding box
         d_real = math.sqrt(z_met_real ** 2 + dic_fin['boxes_3d'][idx][0] ** 2 + dic_fin['boxes_3d'][idx][1] ** 2)
@@ -94,9 +109,9 @@ def update_distances(dic_fin, dic_dist, phase, average_y):
     return cnt
 
 
-def compute_distance(xyz_norm_1, xyz_norm_2, average_y, mode='average', dy_met=0):
+def compute_depth(xyz_norm_1, xyz_norm_2, average_y, mode='average', dy_met=0):
     """
-    Compute distance Z of a mask annotation (solving a linear system) for 2 possible cases:
+    Compute depth Z of a mask annotation (solving a linear system) for 2 possible cases:
     1. knowing specific height of the annotation (head-ankle) dy_met
     2. using mean height of people (average_y)
     """

monoloco/eval/reid_baseline.py

@@ -27,9 +27,9 @@ def get_reid_features(reid_net, boxes, boxes_r, path_image, path_image_r):
     return features.cpu(), features_r.cpu()
 
 
-class ReID(object):
+class ReID:
     def __init__(self, weights_path, device, num_classes=751, height=256, width=128):
-        super(ReID, self).__init__()
+        super().__init__()
         torch.manual_seed(1)
         self.device = device
 
@@ -90,7 +90,7 @@ class ReID(object):
 
 class ResNet50(nn.Module):
     def __init__(self, num_classes, loss):
-        super(ResNet50, self).__init__()
+        super().__init__()
         self.loss = loss
         resnet50 = torchvision.models.resnet50(pretrained=True)
         self.base = nn.Sequential(*list(resnet50.children())[:-2])

monoloco/eval/stereo_baselines.py

@@ -1,12 +1,11 @@
 
 """"Generate stereo baselines for kitti evaluation"""
 
-import warnings
 from collections import defaultdict
 
 import numpy as np
 
-from ..utils import get_keypoints
+from ..utils import get_keypoints, mask_joint_disparity, disparity_to_depth
 
 
 def baselines_association(baselines, zzs, keypoints, keypoints_right, reid_features):
@@ -23,7 +22,7 @@ def baselines_association(baselines, zzs, keypoints, keypoints_right, reid_featu
     cnt_stereo['max'] = min(keypoints.shape[0], keypoints_r.shape[0])  # pylint: disable=E1136
 
     # Filter joints disparity and calculate avg disparity
-    avg_disparities, disparities_x, disparities_y = mask_joint_disparity(keypoints, keypoints_r)
+    avg_disparities, _, _ = mask_joint_disparity(keypoints, keypoints_r)
 
     # Iterate over each left pose
     for key in baselines:
@@ -38,13 +37,14 @@ def baselines_association(baselines, zzs, keypoints, keypoints_right, reid_featu
         best = np.nanmin(similarity)
         while not np.isnan(best):
             idx, arg_best = np.unravel_index(np.nanargmin(similarity), similarity.shape)  # pylint: disable=W0632
-            zz_stereo, flag = similarity_to_depth(avg_disparities[idx, arg_best])
+            zz_stereo, flag = disparity_to_depth(avg_disparities[idx, arg_best])
             zz_mono = zzs[idx]
             similarity[idx, :] = np.nan
             indices_stereo.append(idx)
 
             # Filter stereo depth
-            if flag and verify_stereo(zz_stereo, zz_mono, disparities_x[idx, arg_best], disparities_y[idx, arg_best]):
+            # if flag and verify_stereo(zz_stereo, zz_mono, disparities_x[idx, arg_best], disparities_y[idx, arg_best]):
+            if flag and (1 < zz_stereo < 80):  # Do not add hand-crafted verifications to stereo baselines
                 zzs_stereo[key][idx] = zz_stereo
                 cnt_stereo[key] += 1
                 similarity[:, arg_best] = np.nan
@@ -101,77 +101,3 @@ def features_similarity(features, features_r, key, avg_disparities, zzs):
 
         similarity[idx] = sim_row
     return similarity
-
-
-def similarity_to_depth(avg_disparity):
-
-    try:
-        zz_stereo = 0.54 * 721. / float(avg_disparity)
-        flag = True
-    except (ZeroDivisionError, ValueError):  # All nan-slices or zero division
-        zz_stereo = np.nan
-        flag = False
-
-    return zz_stereo, flag
-
-
-def mask_joint_disparity(keypoints, keypoints_r):
-    """filter joints based on confidence and interquartile range of the distribution"""
-
-    CONF_MIN = 0.3
-    with warnings.catch_warnings() and np.errstate(invalid='ignore'):
-        disparity_x_mask = np.empty((keypoints.shape[0], keypoints_r.shape[0], 17))
-        disparity_y_mask = np.empty((keypoints.shape[0], keypoints_r.shape[0], 17))
-        avg_disparity = np.empty((keypoints.shape[0], keypoints_r.shape[0]))
-
-        for idx, kps in enumerate(keypoints):
-            disparity_x = kps[0, :] - keypoints_r[:, 0, :]
-            disparity_y = kps[1, :] - keypoints_r[:, 1, :]
-
-            # Mask for low confidence
-            mask_conf_left = kps[2, :] > CONF_MIN
-            mask_conf_right = keypoints_r[:, 2, :] > CONF_MIN
-            mask_conf = mask_conf_left & mask_conf_right
-            disparity_x_conf = np.where(mask_conf, disparity_x, np.nan)
-            disparity_y_conf = np.where(mask_conf, disparity_y, np.nan)
-
-            # Mask outliers using iqr
-            mask_outlier = interquartile_mask(disparity_x_conf)
-            x_mask_row = np.where(mask_outlier, disparity_x_conf, np.nan)
-            y_mask_row = np.where(mask_outlier, disparity_y_conf, np.nan)
-            avg_row = np.nanmedian(x_mask_row, axis=1)  # ignore the nan
-
-            # Append
-            disparity_x_mask[idx] = x_mask_row
-            disparity_y_mask[idx] = y_mask_row
-            avg_disparity[idx] = avg_row
-
-        return avg_disparity, disparity_x_mask, disparity_y_mask
-
-
-def verify_stereo(zz_stereo, zz_mono, disparity_x, disparity_y):
-    """Verify disparities based on coefficient of variation, maximum y difference and z difference wrt monoloco"""
-
-    COV_MIN = 0.1
-    y_max_difference = (50 / zz_mono)
-    z_max_difference = 0.6 * zz_mono
-
-    cov = float(np.nanstd(disparity_x) / np.abs(np.nanmean(disparity_x)))  # Coefficient of variation
-    avg_disparity_y = np.nanmedian(disparity_y)
-
-    if abs(zz_stereo - zz_mono) < z_max_difference and \
-            avg_disparity_y < y_max_difference and \
-            cov < COV_MIN\
-            and 4 < zz_stereo < 40:
-        return True
-    # if not np.isnan(zz_stereo):
-    #     return True
-    return False
-
-
-def interquartile_mask(distribution):
-    quartile_1, quartile_3 = np.nanpercentile(distribution, [25, 75], axis=1)
-    iqr = quartile_3 - quartile_1
-    lower_bound = quartile_1 - (iqr * 1.5)
-    upper_bound = quartile_3 + (iqr * 1.5)
-    return (distribution < upper_bound.reshape(-1, 1)) & (distribution > lower_bound.reshape(-1, 1))

monoloco/network/__init__.py

@@ -1,4 +1,3 @@
 
-from .pifpaf import PifPaf, ImageList
+from .net import Loco
-from .losses import LaplacianLoss
+from .process import unnormalize_bi, extract_outputs, extract_labels, extract_labels_aux, extract_labels_cyclist
-from .net import MonoLoco

monoloco/network/architectures.py

@@ -1,15 +1,115 @@
 
+import torch
 import torch.nn as nn
 
 
-class LinearModel(nn.Module):
+class LocoModel(nn.Module):
+
+    def __init__(self, input_size, output_size=2, linear_size=512, p_dropout=0.2, num_stage=3, device='cuda'):
+        super().__init__()
+
+        self.num_stage = num_stage
+        self.stereo_size = input_size
+        self.mono_size = int(input_size / 2)
+        self.output_size = output_size - 1
+        self.linear_size = linear_size
+        self.p_dropout = p_dropout
+        self.num_stage = num_stage
+        self.linear_stages = []
+        self.device = device
+
+        # Initialize weights
+
+        # Preprocessing
+        self.w1 = nn.Linear(self.stereo_size, self.linear_size)
+        self.batch_norm1 = nn.BatchNorm1d(self.linear_size)
+
+        # Internal loop
+        for _ in range(num_stage):
+            self.linear_stages.append(MyLinearSimple(self.linear_size, self.p_dropout))
+        self.linear_stages = nn.ModuleList(self.linear_stages)
+
+        # Post processing
+        self.w2 = nn.Linear(self.linear_size, self.linear_size)
+        self.w3 = nn.Linear(self.linear_size, self.linear_size)
+        self.batch_norm3 = nn.BatchNorm1d(self.linear_size)
+
+        # ------------------------Other----------------------------------------------
+        # Auxiliary
+        self.w_aux = nn.Linear(self.linear_size, 1)
+
+        # Final
+        self.w_fin = nn.Linear(self.linear_size, self.output_size)
+
+        # NO-weight operations
+        self.relu = nn.ReLU(inplace=True)
+        self.dropout = nn.Dropout(self.p_dropout)
+
+    def forward(self, x):
+
+        y = self.w1(x)
+        y = self.batch_norm1(y)
+        y = self.relu(y)
+        y = self.dropout(y)
+
+        for i in range(self.num_stage):
+            y = self.linear_stages[i](y)
+
+        # Auxiliary task
+        y = self.w2(y)
+        aux = self.w_aux(y)
+
+        # Final layers
+        y = self.w3(y)
+        y = self.batch_norm3(y)
+        y = self.relu(y)
+        y = self.dropout(y)
+        y = self.w_fin(y)
+
+        # Cat with auxiliary task
+        y = torch.cat((y, aux), dim=1)
+        return y
+
+
+class MyLinearSimple(nn.Module):
+    def __init__(self, linear_size, p_dropout=0.5):
+        super().__init__()
+        self.l_size = linear_size
+
+        self.relu = nn.ReLU(inplace=True)
+        self.dropout = nn.Dropout(p_dropout)
+
+        self.w1 = nn.Linear(self.l_size, self.l_size)
+        self.batch_norm1 = nn.BatchNorm1d(self.l_size)
+
+        self.w2 = nn.Linear(self.l_size, self.l_size)
+        self.batch_norm2 = nn.BatchNorm1d(self.l_size)
+
+    def forward(self, x):
+
+        y = self.w1(x)
+        y = self.batch_norm1(y)
+        y = self.relu(y)
+        y = self.dropout(y)
+
+        y = self.w2(y)
+        y = self.batch_norm2(y)
+        y = self.relu(y)
+        y = self.dropout(y)
+
+        out = x + y
+
+        return out
+
+
+class MonolocoModel(nn.Module):
     """
     Architecture inspired by https://github.com/una-dinosauria/3d-pose-baseline
     Pytorch implementation from: https://github.com/weigq/3d_pose_baseline_pytorch
     """
 
     def __init__(self, input_size, output_size=2, linear_size=256, p_dropout=0.2, num_stage=3):
-        super(LinearModel, self).__init__()
+        super().__init__()
 
         self.input_size = input_size
         self.output_size = output_size
@@ -23,7 +123,7 @@ class LinearModel(nn.Module):
 
         self.linear_stages = []
         for _ in range(num_stage):
-            self.linear_stages.append(Linear(self.linear_size, self.p_dropout))
+            self.linear_stages.append(MyLinear(self.linear_size, self.p_dropout))
         self.linear_stages = nn.ModuleList(self.linear_stages)
 
         # post processing
@@ -45,9 +145,9 @@ class LinearModel(nn.Module):
         return y
 
 
-class Linear(nn.Module):
+class MyLinear(nn.Module):
     def __init__(self, linear_size, p_dropout=0.5):
-        super(Linear, self).__init__()
+        super().__init__()
         self.l_size = linear_size
 
         self.relu = nn.ReLU(inplace=True)
@@ -60,6 +160,7 @@ class Linear(nn.Module):
         self.batch_norm2 = nn.BatchNorm1d(self.l_size)
 
     def forward(self, x):
+
         y = self.w1(x)
         y = self.batch_norm1(y)
         y = self.relu(y)

monoloco/network/losses.py

@@ -1,141 +0,0 @@
-
-import math
-import torch
-import numpy as np
-import matplotlib.pyplot as plt
-
-
-class CustomL1Loss(torch.nn.Module):
-    """
-    L1 loss with more weight to errors at a shorter distance
-    It inherits from nn.module so it supports backward
-    """
-
-    def __init__(self, dic_norm, device, beta=1):
-        super(CustomL1Loss, self).__init__()
-
-        self.dic_norm = dic_norm
-        self.device = device
-        self.beta = beta
-
-    @staticmethod
-    def compute_weights(xx, beta=1):
-        """
-        Return the appropriate weight depending on the distance and the hyperparameter chosen
-        alpha = 1 refers to the curve of A Photogrammetric Approach for Real-time...
-        It is made for unnormalized outputs (to be more understandable)
-        From 70 meters on every value is weighted the same (0.1**beta)
-        Alpha is optional value from Focal loss. Yet to be analyzed
-        """
-        # alpha = np.maximum(1, 10 ** (beta - 1))
-        alpha = 1
-        ww = np.maximum(0.1, 1 - xx / 78)**beta
-
-        return alpha * ww
-
-    def print_loss(self):
-        xx = np.linspace(0, 80, 100)
-        y1 = self.compute_weights(xx, beta=1)
-        y2 = self.compute_weights(xx, beta=2)
-        y3 = self.compute_weights(xx, beta=3)
-        plt.plot(xx, y1)
-        plt.plot(xx, y2)
-        plt.plot(xx, y3)
-        plt.xlabel("Distance [m]")
-        plt.ylabel("Loss function Weight")
-        plt.legend(("Beta = 1", "Beta = 2", "Beta = 3"))
-        plt.show()
-
-    def forward(self, output, target):
-
-        unnormalized_output = output.cpu().detach().numpy() * self.dic_norm['std']['Y'] + self.dic_norm['mean']['Y']
-        weights_np = self.compute_weights(unnormalized_output, self.beta)
-        weights = torch.from_numpy(weights_np).float().to(self.device)  # To make weights in the same cuda device
-        losses = torch.abs(output - target) * weights
-        loss = losses.mean()  # Mean over the batch
-        return loss
-
-
-class LaplacianLoss(torch.nn.Module):
-    """1D Gaussian with std depending on the absolute distance
-    """
-    def __init__(self, size_average=True, reduce=True, evaluate=False):
-        super(LaplacianLoss, self).__init__()
-        self.size_average = size_average
-        self.reduce = reduce
-        self.evaluate = evaluate
-
-    def laplacian_1d(self, mu_si, xx):
-        """
-        1D Gaussian Loss. f(x | mu, sigma). The network outputs mu and sigma. X is the ground truth distance.
-        This supports backward().
-        Inspired by
-        https://github.com/naba89/RNN-Handwriting-Generation-Pytorch/blob/master/loss_functions.py
-
-        """
-        mu, si = mu_si[:, 0:1], mu_si[:, 1:2]
-        # norm = xx - mu
-        norm = 1 - mu / xx  # Relative
-
-        term_a = torch.abs(norm) * torch.exp(-si)
-        term_b = si
-        norm_bi = (np.mean(np.abs(norm.cpu().detach().numpy())), np.mean(torch.exp(si).cpu().detach().numpy()))
-
-        if self.evaluate:
-            return norm_bi
-        return term_a + term_b
-
-    def forward(self, outputs, targets):
-
-        values = self.laplacian_1d(outputs, targets)
-
-        if not self.reduce or self.evaluate:
-            return values
-        if self.size_average:
-            mean_values = torch.mean(values)
-            return mean_values
-        return torch.sum(values)
-
-
-class GaussianLoss(torch.nn.Module):
-    """1D Gaussian with std depending on the absolute distance
-    """
-    def __init__(self, device, size_average=True, reduce=True, evaluate=False):
-        super(GaussianLoss, self).__init__()
-        self.size_average = size_average
-        self.reduce = reduce
-        self.evaluate = evaluate
-        self.device = device
-
-    def gaussian_1d(self, mu_si, xx):
-        """
-        1D Gaussian Loss. f(x | mu, sigma). The network outputs mu and sigma. X is the ground truth distance.
-        This supports backward().
-        Inspired by
-        https://github.com/naba89/RNN-Handwriting-Generation-Pytorch/blob/master/loss_functions.py
-        """
-        mu, si = mu_si[:, 0:1], mu_si[:, 1:2]
-
-        min_si = torch.ones(si.size()).cuda(self.device) * 0.1
-        si = torch.max(min_si, si)
-        norm = xx - mu
-        term_a = (norm / si)**2 / 2
-        term_b = torch.log(si * math.sqrt(2 * math.pi))
-
-        norm_si = (np.mean(np.abs(norm.cpu().detach().numpy())), np.mean(si.cpu().detach().numpy()))
-
-        if self.evaluate:
-            return norm_si
-
-        return term_a + term_b
-
-    def forward(self, outputs, targets):
-
-        values = self.gaussian_1d(outputs, targets)
-
-        if not self.reduce or self.evaluate:
-            return values
-        if self.size_average:
-            mean_values = torch.mean(values)
-            return mean_values
-        return torch.sum(values)

monoloco/network/net.py

@@ -1,27 +1,76 @@
+# pylint: disable=too-many-statements, too-many-branches
 
 """
-Monoloco class. From 2D joints to real-world distances
+Loco super class for MonStereo, MonoLoco, MonoLoco++ nets.
+From 2D joints to real-world distances with monocular &/or stereo cameras
 """
 
+import math
 import logging
 from collections import defaultdict
 
+import numpy as np
 import torch
 
-from ..utils import get_iou_matches, reorder_matches, get_keypoints, pixel_to_camera, xyz_from_distance
+from ..utils import get_iou_matches, reorder_matches, get_keypoints, pixel_to_camera, xyz_from_distance,  \
-from .process import preprocess_monoloco, unnormalize_bi, laplace_sampling
+    mask_joint_disparity
-from .architectures import LinearModel
+from .process import preprocess_monstereo, preprocess_monoloco, extract_outputs, extract_outputs_mono,\
+    filter_outputs, cluster_outputs, unnormalize_bi, laplace_sampling
+from ..activity import social_interactions, is_raising_hand, is_phoning, is_turning
+from .architectures import MonolocoModel, LocoModel
 
 
-class MonoLoco:
+class Loco:
-
+    """Class for both MonoLoco and MonStereo"""
     logging.basicConfig(level=logging.INFO)
     logger = logging.getLogger(__name__)
-    INPUT_SIZE = 17 * 2
+    LINEAR_SIZE_MONO = 256
-    LINEAR_SIZE = 256
     N_SAMPLES = 100
 
-    def __init__(self, model, device=None, n_dropout=0, p_dropout=0.2):
+    def __init__(self, model, mode, net=None, device=None, n_dropout=0,
+                 p_dropout=0.2, linear_size=1024, casr=None, casr_model=None):
+
+        # Select networks
+        assert mode in ('mono', 'stereo'), "mode not recognized"
+        self.mode = mode
+        if net is None:
+            if mode == 'mono':
+                self.net = 'monoloco_pp'
+            else:
+                self.net = 'monstereo'
+        else:
+            assert self.net in ('monstereo', 'monoloco', 'monoloco_p', 'monoloco_pp')
+            if self.net != 'monstereo':
+                assert mode == 'stereo', "Assert arguments mode and net are in conflict"
+            self.net = net
+
+        if self.net == 'monstereo':
+            input_size = 68
+            output_size = 10
+        elif self.net == 'monoloco_p':
+            input_size = 34
+            output_size = 9
+            linear_size = 256
+        elif self.net == 'monoloco_pp':
+            input_size = 34
+            output_size = 9
+        else:
+            input_size = 34
+            output_size = 2
+
+        if casr == 'std':
+            print("CASR with standard gestures")
+            turning_output_size = 3
+            if casr_model:
+                turning_model_path = casr_model
+            else:
+                turning_model_path = "/home/beauvill/Repos/monoloco/data/outputs/casr_standard-210613-0005.pkl"
+        elif casr == 'nonstd':
+            turning_output_size = 4
+            if casr_model:
+                turning_model_path = casr_model
+            else:
+                turning_model_path = "/home/beauvill/Repos/monoloco/data/outputs/casr-210615-1128.pkl"
 
         if not device:
             self.device = torch.device('cpu')
@@ -33,86 +82,270 @@ class MonoLoco:
         # if the path is provided load the model parameters
         if isinstance(model, str):
             model_path = model
-            self.model = LinearModel(p_dropout=p_dropout, input_size=self.INPUT_SIZE, linear_size=self.LINEAR_SIZE)
+            if net in ('monoloco', 'monoloco_p'):
-            self.model.load_state_dict(torch.load(model_path, map_location=lambda storage, loc: storage))
+                self.model = MonolocoModel(p_dropout=p_dropout, input_size=input_size, linear_size=linear_size,
+                                           output_size=output_size)
+                if casr:
+                    self.turning_model = MonolocoModel(p_dropout=p_dropout, input_size=34, linear_size=linear_size,
+                                                       output_size=turning_output_size)
+            else:
+                self.model = LocoModel(p_dropout=p_dropout, input_size=input_size, output_size=output_size,
+                                            linear_size=linear_size, device=self.device)
+                if casr:
+                    self.turning_model = LocoModel(p_dropout=p_dropout, input_size=34, output_size=turning_output_size,
+                                                   linear_size=linear_size, device=self.device)
 
-        # if the model is directly provided
+            self.model.load_state_dict(torch.load(model_path, map_location=lambda storage, loc: storage))
+            if casr:
+                self.turning_model.load_state_dict(torch.load(turning_model_path,
+                                                              map_location=lambda storage, loc: storage))
         else:
             self.model = model
         self.model.eval()  # Default is train
         self.model.to(self.device)
+        if casr:
+            self.turning_model.eval()  # Default is train
+            self.turning_model.to(self.device)
 
-    def forward(self, keypoints, kk):
+    def forward(self, keypoints, kk, keypoints_r=None):
-        """forward pass of monoloco network"""
+        """
+        Forward pass of MonSter or monoloco network
+        It includes preprocessing and postprocessing of data
+        """
         if not keypoints:
-            return None, None
+            return None
 
         with torch.no_grad():
-            inputs = preprocess_monoloco(torch.tensor(keypoints).to(self.device), torch.tensor(kk).to(self.device))
+            keypoints = torch.tensor(keypoints).to(self.device)
-            if self.n_dropout > 0:
+            kk = torch.tensor(kk).to(self.device)
+
+            if self.net == 'monoloco':
+                inputs = preprocess_monoloco(keypoints, kk, zero_center=True)
+                outputs = self.model(inputs)
+                bi = unnormalize_bi(outputs)
+                dic_out = {'d': outputs[:, 0:1], 'bi': bi}
+                dic_out = {key: el.detach().cpu() for key, el in dic_out.items()}
+
+            elif self.net == 'monoloco_p':
+                inputs = preprocess_monoloco(keypoints, kk)
+                outputs = self.model(inputs)
+                dic_out = extract_outputs_mono(outputs)
+
+            elif self.net == 'monoloco_pp':
+                inputs = preprocess_monoloco(keypoints, kk)
+                outputs = self.model(inputs)
+                dic_out = extract_outputs(outputs)
+
+            else:
+                if keypoints_r:
+                    keypoints_r = torch.tensor(keypoints_r).to(self.device)
+                else:
+                    keypoints_r = keypoints[0:1, :].clone()
+                inputs, _ = preprocess_monstereo(keypoints, keypoints_r, kk)
+                outputs = self.model(inputs)
+
+                outputs = cluster_outputs(outputs, keypoints_r.shape[0])
+                outputs_fin, _ = filter_outputs(outputs)
+                dic_out = extract_outputs(outputs_fin)
+
+                # For Median baseline
+                # dic_out = median_disparity(dic_out, keypoints, keypoints_r, mask)
+
+            if self.n_dropout > 0 and self.net != 'monstereo':
+                varss = self.epistemic_uncertainty(inputs)
+                dic_out['epi'] = varss
+            else:
+                dic_out['epi'] = [0.] * outputs.shape[0]
+                # Add in the dictionary
+
+        return dic_out
+
+    def epistemic_uncertainty(self, inputs):
+        """
+        Apply dropout at test time to obtain combined aleatoric + epistemic uncertainty
+        """
+        assert self.net in ('monoloco', 'monoloco_p', 'monoloco_pp'), "Not supported for MonStereo"
+
         self.model.dropout.training = True  # Manually reactivate dropout in eval
         total_outputs = torch.empty((0, inputs.size()[0])).to(self.device)
 
         for _ in range(self.n_dropout):
             outputs = self.model(inputs)
-                    outputs = unnormalize_bi(outputs)
+
+            # Extract localization output
+            if self.net == 'monoloco':
+                db = outputs[:, 0:2]
+            else:
+                db = outputs[:, 2:4]
+
+            # Unnormalize b and concatenate
+            bi = unnormalize_bi(db)
+            outputs = torch.cat((db[:, 0:1], bi), dim=1)
+
             samples = laplace_sampling(outputs, self.N_SAMPLES)
             total_outputs = torch.cat((total_outputs, samples), 0)
         varss = total_outputs.std(0)
         self.model.dropout.training = False
-            else:
+        return varss
-                varss = torch.zeros(inputs.size()[0])
-
-            #  Don't use dropout for the mean prediction
-            outputs = self.model(inputs)
-            outputs = unnormalize_bi(outputs)
-        return outputs, varss
 
     @staticmethod
-    def post_process(outputs, varss, boxes, keypoints, kk, dic_gt=None, iou_min=0.3):
+    def post_process(dic_in, boxes, keypoints, kk, dic_gt=None, iou_min=0.3, reorder=True, verbose=False):
         """Post process monoloco to output final dictionary with all information for visualizations"""
 
         dic_out = defaultdict(list)
-        if outputs is None:
+        if dic_in is None:
             return dic_out
 
         if dic_gt:
-            boxes_gt, dds_gt = dic_gt['boxes'], dic_gt['dds']
+            boxes_gt = dic_gt['boxes']
-            matches = get_iou_matches(boxes, boxes_gt, thresh=iou_min)
+            dds_gt = [el[3] for el in dic_gt['ys']]
+            matches = get_iou_matches(boxes, boxes_gt, iou_min=iou_min)
+            dic_out['gt'] = [True]
+            if verbose:
                 print("found {} matches with ground-truth".format(len(matches)))
-        else:
-            matches = [(idx, idx) for idx, _ in enumerate(boxes)]  # Replicate boxes
 
+            # Keep track of instances non-matched
+            idxs_matches = [el[0] for el in matches]
+            not_matches = [idx for idx, _ in enumerate(boxes) if idx not in idxs_matches]
+
+        else:
+            matches = []
+            not_matches = list(range(len(boxes)))
+            if verbose:
+                print("NO ground-truth associated")
+
+        if reorder and matches:
             matches = reorder_matches(matches, boxes, mode='left_right')
+
+        all_idxs = [idx for idx, _ in matches] + not_matches
+        dic_out['gt'] = [True]*len(matches) + [False]*len(not_matches)
+
         uv_shoulders = get_keypoints(keypoints, mode='shoulder')
+        uv_heads = get_keypoints(keypoints, mode='head')
         uv_centers = get_keypoints(keypoints, mode='center')
         xy_centers = pixel_to_camera(uv_centers, kk, 1)
 
-        # Match with ground truth if available
+        # Add all the predicted annotations, starting with the ones that match a ground-truth
-        for idx, idx_gt in matches:
+        for idx in all_idxs:
-            dd_pred = float(outputs[idx][0])
-            ale = float(outputs[idx][1])
-            var_y = float(varss[idx])
-            dd_real = dds_gt[idx_gt] if dic_gt else dd_pred
-
             kps = keypoints[idx]
             box = boxes[idx]
+            dd_pred = float(dic_in['d'][idx])
+            bi = float(dic_in['bi'][idx])
+            var_y = float(dic_in['epi'][idx])
             uu_s, vv_s = uv_shoulders.tolist()[idx][0:2]
             uu_c, vv_c = uv_centers.tolist()[idx][0:2]
+            uu_h, vv_h = uv_heads.tolist()[idx][0:2]
             uv_shoulder = [round(uu_s), round(vv_s)]
             uv_center = [round(uu_c), round(vv_c)]
-            xyz_real = xyz_from_distance(dd_real, xy_centers[idx])
+            uv_head = [round(uu_h), round(vv_h)]
-            xyz_pred = xyz_from_distance(dd_pred, xy_centers[idx])
+            xyz_pred = xyz_from_distance(dd_pred, xy_centers[idx])[0]
+            distance = math.sqrt(float(xyz_pred[0])**2 + float(xyz_pred[1])**2 + float(xyz_pred[2])**2)
+            conf = 0.035 * (box[-1]) / (bi / distance)
+
             dic_out['boxes'].append(box)
-            dic_out['boxes_gt'].append(boxes_gt[idx_gt] if dic_gt else boxes[idx])
+            dic_out['confs'].append(conf)
-            dic_out['dds_real'].append(dd_real)
             dic_out['dds_pred'].append(dd_pred)
-            dic_out['stds_ale'].append(ale)
+            dic_out['stds_ale'].append(bi)
             dic_out['stds_epi'].append(var_y)
-            dic_out['xyz_real'].append(xyz_real.squeeze().tolist())
+
             dic_out['xyz_pred'].append(xyz_pred.squeeze().tolist())
             dic_out['uv_kps'].append(kps)
             dic_out['uv_centers'].append(uv_center)
             dic_out['uv_shoulders'].append(uv_shoulder)
+            dic_out['uv_heads'].append(uv_head)
+
+            # For MonStereo / MonoLoco++
+            try:
+                dic_out['angles'].append(float(dic_in['yaw'][0][idx]))  # Predicted angle
+                dic_out['angles_egocentric'].append(float(dic_in['yaw'][1][idx]))  # Egocentric angle
+            except KeyError:
+                continue
+
+            # Only for MonStereo
+            try:
+                dic_out['aux'].append(float(dic_in['aux'][idx]))
+            except KeyError:
+                continue
+
+        for idx, idx_gt in matches:
+            dd_real = dds_gt[idx_gt]
+            xyz_real = xyz_from_distance(dd_real, xy_centers[idx])
+            dic_out['dds_real'].append(dd_real)
+            dic_out['boxes_gt'].append(boxes_gt[idx_gt])
+            dic_out['xyz_real'].append(xyz_real.squeeze().tolist())
+        return dic_out
+
+    @staticmethod
+    def social_distance(dic_out, args):
+
+        angles = dic_out['angles']
+        dds = dic_out['dds_pred']
+        stds = dic_out['stds_ale']
+        xz_centers = [[xx[0], xx[2]] for xx in dic_out['xyz_pred']]
+
+        # Prepare color for social distancing
+        dic_out['social_distance'] = [bool(social_interactions(idx, xz_centers, angles, dds,
+                                                               stds=stds,
+                                                               threshold_prob=args.threshold_prob,
+                                                               threshold_dist=args.threshold_dist,
+                                                               radii=args.radii))
+                                      for idx, _ in enumerate(dic_out['xyz_pred'])]
+        return dic_out
+
+
+    @staticmethod
+    def raising_hand(dic_out, keypoints):
+        dic_out['raising_hand'] = [is_raising_hand(keypoint) for keypoint in keypoints]
+        return dic_out
+
+    @staticmethod
+    def using_phone(dic_out, keypoints):
+        dic_out['using_phone'] = [is_phoning(keypoint) for keypoint in keypoints]
+        return dic_out
+
+    @staticmethod
+    def turning(dic_out, keypoints):
+        dic_out['turning'] = [is_turning(keypoint) for keypoint in keypoints]
+        return dic_out
+
+    def turning_forward(self, dic_out, keypoints):
+        """
+        Forward pass of MonSter or monoloco network
+        It includes preprocessing and postprocessing of data
+        """
+        if not keypoints:
+            return None
+
+        with torch.no_grad():
+            keypoints = torch.tensor(keypoints).to(self.device)
+            kk = torch.eye(3).to(self.device)
+
+            inputs = preprocess_monoloco(keypoints, kk, zero_center=False)
+            outputs = self.turning_model(inputs)
+            # bi = unnormalize_bi(outputs)
+            dic = {'turning': torch.argmax(outputs, axis=len(outputs.shape)-1).tolist()}
+            # dic = {key: el.detach().cpu() for key, el in dic.items()}
+            dic_out['turning'] = dic['turning']
 
         return dic_out
+
+def median_disparity(dic_out, keypoints, keypoints_r, mask):
+    """
+    Ablation study: whenever a matching is found, compute depth by median disparity instead of using MonSter
+    Filters are applied to masks nan joints and remove outlier disparities with iqr
+    The mask input is used to filter the all-vs-all approach
+    """
+
+    keypoints = keypoints.cpu().numpy()
+    keypoints_r = keypoints_r.cpu().numpy()
+    mask = mask.cpu().numpy()
+    avg_disparities, _, _ = mask_joint_disparity(keypoints, keypoints_r)
+    BF = 0.54 * 721
+    for idx, aux in enumerate(dic_out['aux']):
+        if aux > 0.5:
+            idx_r = np.argmax(mask[idx])
+            z = BF / avg_disparities[idx][idx_r]
+            if 1 < z < 80:
+                dic_out['xyzd'][idx][2] = z
+                dic_out['xyzd'][idx][3] = torch.norm(dic_out['xyzd'][idx][0:3])
+    return dic_out

monoloco/network/pifpaf.py

@@ -1,105 +0,0 @@
-
-import glob
-
-import numpy as np
-import torchvision
-import torch
-from PIL import Image, ImageFile
-from openpifpaf.network import nets
-from openpifpaf import decoder
-
-from .process import image_transform
-
-
-class ImageList(torch.utils.data.Dataset):
-    """It defines transformations to apply to images and outputs of the dataloader"""
-    def __init__(self, image_paths, scale):
-        self.image_paths = image_paths
-        self.scale = scale
-
-    def __getitem__(self, index):
-        image_path = self.image_paths[index]
-        ImageFile.LOAD_TRUNCATED_IMAGES = True
-        with open(image_path, 'rb') as f:
-            image = Image.open(f).convert('RGB')
-
-        if self.scale > 1.01 or self.scale < 0.99:
-            image = torchvision.transforms.functional.resize(image,
-                                                             (round(self.scale * image.size[1]),
-                                                              round(self.scale * image.size[0])),
-                                                             interpolation=Image.BICUBIC)
-        # PIL images are not iterables
-        original_image = torchvision.transforms.functional.to_tensor(image)  # 0-255 --> 0-1
-        image = image_transform(image)
-
-        return image_path, original_image, image
-
-    def __len__(self):
-        return len(self.image_paths)
-
-
-def factory_from_args(args):
-
-    # Merge the model_pifpaf argument
-    if not args.checkpoint:
-        args.checkpoint = 'resnet152'  # Default model Resnet 152
-    # glob
-    if not args.webcam:
-        if args.glob:
-            args.images += glob.glob(args.glob)
-        if not args.images:
-            raise Exception("no image files given")
-
-    # add args.device
-    args.device = torch.device('cpu')
-    args.pin_memory = False
-    if torch.cuda.is_available():
-        args.device = torch.device('cuda')
-        args.pin_memory = True
-
-    # Add num_workers
-    args.loader_workers = 8
-
-    # Add visualization defaults
-    args.figure_width = 10
-    args.dpi_factor = 1.0
-
-    return args
-
-
-class PifPaf:
-    def __init__(self, args):
-        """Instanciate the mdodel"""
-        factory_from_args(args)
-        model_pifpaf, _ = nets.factory_from_args(args)
-        model_pifpaf = model_pifpaf.to(args.device)
-        self.processor = decoder.factory_from_args(args, model_pifpaf)
-        self.keypoints_whole = []
-
-        # Scale the keypoints to the original image size for printing (if not webcam)
-        if not args.webcam:
-            self.scale_np = np.array([args.scale, args.scale, 1] * 17).reshape(17, 3)
-        else:
-            self.scale_np = np.array([1, 1, 1] * 17).reshape(17, 3)
-
-    def fields(self, processed_images):
-        """Encoder for pif and paf fields"""
-        fields_batch = self.processor.fields(processed_images)
-        return fields_batch
-
-    def forward(self, image, processed_image_cpu, fields):
-        """Decoder, from pif and paf fields to keypoints"""
-        self.processor.set_cpu_image(image, processed_image_cpu)
-        keypoint_sets, scores = self.processor.keypoint_sets(fields)
-
-        if keypoint_sets.size > 0:
-            self.keypoints_whole.append(np.around((keypoint_sets / self.scale_np), 1)
-                                        .reshape(keypoint_sets.shape[0], -1).tolist())
-
-        pifpaf_out = [
-            {'keypoints': np.around(kps / self.scale_np, 1).reshape(-1).tolist(),
-             'bbox': [np.min(kps[:, 0]) / self.scale_np[0, 0], np.min(kps[:, 1]) / self.scale_np[0, 0],
-                      np.max(kps[:, 0]) / self.scale_np[0, 0], np.max(kps[:, 1]) / self.scale_np[0, 0]]}
-            for kps in keypoint_sets
-        ]
-        return keypoint_sets, scores, pifpaf_out

monoloco/network/process.py

@@ -1,14 +1,49 @@
 
 import json
+import os
+import logging
 
 import numpy as np
 import torch
 import torchvision
 
-from ..utils import get_keypoints, pixel_to_camera
+from ..utils import get_keypoints, pixel_to_camera, to_cartesian, back_correct_angles, open_annotations
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+BF = 0.54 * 721
+z_min = 4
+z_max = 60
+D_MIN = BF / z_max
+D_MAX = BF / z_min
+Sx = 7.2  # nuScenes sensor size (mm)
+Sy = 5.4  # nuScenes sensor size (mm)
 
 
-def preprocess_monoloco(keypoints, kk):
+def preprocess_monstereo(keypoints, keypoints_r, kk):
+    """
+    Combine left and right keypoints in all-vs-all settings
+    """
+    clusters = []
+    inputs_l = preprocess_monoloco(keypoints, kk)
+    inputs_r = preprocess_monoloco(keypoints_r, kk)
+
+    inputs = torch.empty((0, 68)).to(inputs_l.device)
+    for inp_l in inputs_l.split(1):
+        clst = 0
+        # inp_l = torch.cat((inp_l, cat[:, idx:idx+1]), dim=1)
+        for idx_r, inp_r in enumerate(inputs_r.split(1)):
+            # if D_MIN < avg_disparities[idx_r] < D_MAX:  # Check the range of disparities
+            inp_r = inputs_r[idx_r, :]
+            inp = torch.cat((inp_l, inp_l - inp_r), dim=1)  # (1,68)
+            inputs = torch.cat((inputs, inp), dim=0)
+            clst += 1
+        clusters.append(clst)
+    return inputs, clusters
+
+
+def preprocess_monoloco(keypoints, kk, zero_center=False):
 
     """ Preprocess batches of inputs
     keypoints = torch tensors of (m, 3, 17)  or list [3,17]
@@ -22,46 +57,44 @@ def preprocess_monoloco(keypoints, kk):
     uv_center = get_keypoints(keypoints, mode='center')
     xy1_center = pixel_to_camera(uv_center, kk, 10)
     xy1_all = pixel_to_camera(keypoints[:, 0:2, :], kk, 10)
+    if zero_center:
         kps_norm = xy1_all - xy1_center.unsqueeze(1)  # (m, 17, 3) - (m, 1, 3)
+    else:
+        kps_norm = xy1_all
     kps_out = kps_norm[:, :, 0:2].reshape(kps_norm.size()[0], -1)  # no contiguous for view
+    # kps_out = torch.cat((kps_out, keypoints[:, 2, :]), dim=1)
     return kps_out
 
 
-def factory_for_gt(im_size, name=None, path_gt=None):
+def factory_for_gt(im_size, focal_length=5.7, name=None, path_gt=None):
     """Look for ground-truth annotations file and define calibration matrix based on image size """
 
-    try:
+    if path_gt is not None:
+        assert os.path.exists(path_gt), "Ground-truth file not found"
         with open(path_gt, 'r') as f:
             dic_names = json.load(f)
-        print('-' * 120 + "\nGround-truth file opened")
-    except (FileNotFoundError, TypeError):
-        print('-' * 120 + "\nGround-truth file not found")
-        dic_names = {}
-
-    try:
             kk = dic_names[name]['K']
             dic_gt = dic_names[name]
-        print("Matched ground-truth file!")
-    except KeyError:
-        dic_gt = None
-        x_factor = im_size[0] / 1600
-        y_factor = im_size[1] / 900
-        pixel_factor = (x_factor + y_factor) / 2   # TODO remove and check it
-        if im_size[0] / im_size[1] > 2.5:
-            kk = [[718.3351, 0., 600.3891], [0., 718.3351, 181.5122], [0., 0., 1.]]  # Kitti calibration
-        else:
-            kk = [[1266.4 * pixel_factor, 0., 816.27 * x_factor],
-                  [0, 1266.4 * pixel_factor, 491.5 * y_factor],
-                  [0., 0., 1.]]  # nuScenes calibration
 
-        print("Using a standard calibration matrix...")
+    # Without ground-truth-file
+    elif im_size[0] / im_size[1] > 2.5:  # KITTI default
+        kk = [[718.3351, 0., 600.3891], [0., 718.3351, 181.5122], [0., 0., 1.]]  # Kitti calibration
+        dic_gt = None
+        logger.info("Using KITTI calibration matrix...")
+    else:  # nuScenes camera parameters
+        kk = [
+            [im_size[0]*focal_length/Sx, 0., im_size[0]/2],
+            [0., im_size[1]*focal_length/Sy, im_size[1]/2],
+            [0., 0., 1.]]
+        dic_gt = None
+        logger.info("Using a standard calibration matrix...")
 
     return kk, dic_gt
 
 
 def laplace_sampling(outputs, n_samples):
 
-    # np.random.seed(1)
+    torch.manual_seed(1)
     mu = outputs[:, 0]
     bi = torch.abs(outputs[:, 1])
 
@@ -83,14 +116,37 @@ def laplace_sampling(outputs, n_samples):
     return xx
 
 
-def unnormalize_bi(outputs):
+def unnormalize_bi(loc):
-    """Unnormalize relative bi of a nunmpy array"""
+    """
+    Unnormalize relative bi of a nunmpy array
+    Input --> tensor of (m, 2)
+    """
+    assert loc.size()[1] == 2, "size of the output tensor should be (m, 2)"
+    bi = torch.exp(loc[:, 1:2]) * loc[:, 0:1]
 
-    outputs[:, 1] = torch.exp(outputs[:, 1]) * outputs[:, 0]
+    return bi
-    return outputs
 
 
-def preprocess_pifpaf(annotations, im_size=None):
+def preprocess_mask(dir_ann, basename, mode='left'):
+
+    dir_ann = os.path.join(os.path.split(dir_ann)[0], 'mask')
+    if mode == 'left':
+        path_ann = os.path.join(dir_ann, basename + '.json')
+    elif mode == 'right':
+        path_ann = os.path.join(dir_ann + '_right', basename + '.json')
+
+    dic = open_annotations(path_ann)
+    if isinstance(dic, list):
+        return [], []
+
+    keypoints = []
+    for kps in dic['keypoints']:
+        kps = prepare_pif_kps(np.array(kps).reshape(51,).tolist())
+        keypoints.append(kps)
+    return dic['boxes'], keypoints
+
+
+def preprocess_pifpaf(annotations, im_size=None, enlarge_boxes=True, min_conf=0.):
     """
     Preprocess pif annotations:
     1. enlarge the box of 10%
@@ -99,19 +155,32 @@ def preprocess_pifpaf(annotations, im_size=None):
 
     boxes = []
     keypoints = []
+    enlarge = 1 if enlarge_boxes else 2  # Avoid enlarge boxes for social distancing
 
     for dic in annotations:
-        box = dic['bbox']
-        if box[3] < 0.5:  # Check for no detections (boxes 0,0,0,0)
-            return [], []
-
         kps = prepare_pif_kps(dic['keypoints'])
-        conf = float(np.sort(np.array(kps[2]))[-3])  # The confidence is the 3rd highest value for the keypoints
+        box = dic['bbox']
+        try:
+            conf = dic['score']
+            # Enlarge boxes
+            delta_h = (box[3]) / (10 * enlarge)
+            delta_w = (box[2]) / (5 * enlarge)
+            # from width height to corners
+            box[2] += box[0]
+            box[3] += box[1]
 
+        except KeyError:
+            all_confs = np.array(kps[2])
+            score_weights = np.ones(17)
+            score_weights[:3] = 3.0
+            score_weights[5:] = 0.1
+            # conf = np.sum(score_weights * np.sort(all_confs)[::-1])
+            conf = float(np.mean(all_confs))
             # Add 15% for y and 20% for x
-        delta_h = (box[3] - box[1]) / 7
+            delta_h = (box[3] - box[1]) / (7 * enlarge)
-        delta_w = (box[2] - box[0]) / 3.5
+            delta_w = (box[2] - box[0]) / (3.5 * enlarge)
             assert delta_h > -5 and delta_w > -5, "Bounding box <=0"
+
         box[0] -= delta_w
         box[1] -= delta_h
         box[2] += delta_w
@@ -124,6 +193,7 @@ def preprocess_pifpaf(annotations, im_size=None):
             box[2] = min(box[2], im_size[0])
             box[3] = min(box[3], im_size[1])
 
+        if conf >= min_conf:
             box.append(conf)
             boxes.append(box)
             keypoints.append(kps)
@@ -150,3 +220,146 @@ def image_transform(image):
     )
     transforms = torchvision.transforms.Compose([torchvision.transforms.ToTensor(), normalize, ])
     return transforms(image)
+
+
+def extract_outputs(outputs, tasks=()):
+    """
+    Extract the outputs for multi-task training and predictions
+    Inputs:
+        tensor (m, 10) or (m,9) if monoloco
+    Outputs:
+         - if tasks are provided return ordered list of raw tensors
+         - else return a dictionary with processed outputs
+    """
+    dic_out = {'x': outputs[:, 0:1],
+               'y': outputs[:, 1:2],
+               'd': outputs[:, 2:4],
+               'h': outputs[:, 4:5],
+               'w': outputs[:, 5:6],
+               'l': outputs[:, 6:7],
+               'ori': outputs[:, 7:9],
+               'cyclist': outputs}
+
+    if outputs.shape[1] == 10:
+        dic_out['aux'] = outputs[:, 9:10]
+
+    # Multi-task training
+    if len(tasks) >= 1:
+        assert isinstance(tasks, tuple), "tasks need to be a tuple"
+        return [dic_out[task] for task in tasks]
+
+    # Preprocess the tensor
+    # AV_H, AV_W, AV_L, HWL_STD = 1.72, 0.75, 0.68, 0.1
+    bi = unnormalize_bi(dic_out['d'])
+    dic_out['bi'] = bi
+
+    dic_out = {key: el.detach().cpu() for key, el in dic_out.items()}
+    x = to_cartesian(outputs[:, 0:3].detach().cpu(), mode='x')
+    y = to_cartesian(outputs[:, 0:3].detach().cpu(), mode='y')
+    d = dic_out['d'][:, 0:1]
+    z = torch.sqrt(d**2 - x**2 - y**2)
+    dic_out['xyzd'] = torch.cat((x, y, z, d), dim=1)
+    dic_out.pop('d')
+    dic_out.pop('x')
+    dic_out.pop('y')
+    dic_out['d'] = d
+
+    yaw_pred = torch.atan2(dic_out['ori'][:, 0:1], dic_out['ori'][:, 1:2])
+    yaw_orig = back_correct_angles(yaw_pred, dic_out['xyzd'][:, 0:3])
+    dic_out['yaw'] = (yaw_pred, yaw_orig)  # alpha, ry
+
+    if outputs.shape[1] == 10:
+        dic_out['aux'] = torch.sigmoid(dic_out['aux'])
+    return dic_out
+
+
+def extract_labels_aux(labels, tasks=None):
+
+    dic_gt_out = {'aux': labels[:, 0:1]}
+
+    if tasks is not None:
+        assert isinstance(tasks, tuple), "tasks need to be a tuple"
+        return [dic_gt_out[task] for task in tasks]
+
+    dic_gt_out = {key: el.detach().cpu() for key, el in dic_gt_out.items()}
+    return dic_gt_out
+
+def extract_labels_cyclist(labels, tasks=None):
+
+    dic_gt_out = {'cyclist': labels}
+
+    if tasks is not None:
+        assert isinstance(tasks, tuple), "tasks need to be a tuple"
+        return [dic_gt_out[task] for task in tasks]
+
+    dic_gt_out = {key: el.detach().cpu() for key, el in dic_gt_out.items()}
+    return dic_gt_out
+
+def extract_labels(labels, tasks=None):
+
+    dic_gt_out = {'x': labels[:, 0:1], 'y': labels[:, 1:2], 'z': labels[:, 2:3], 'd': labels[:, 3:4],
+                  'h': labels[:, 4:5], 'w': labels[:, 5:6], 'l': labels[:, 6:7],
+                  'ori': labels[:, 7:9], 'aux': labels[:, 10:11]}
+
+    if tasks is not None:
+        assert isinstance(tasks, tuple), "tasks need to be a tuple"
+        return [dic_gt_out[task] for task in tasks]
+
+    dic_gt_out = {key: el.detach().cpu() for key, el in dic_gt_out.items()}
+    return dic_gt_out
+
+
+def cluster_outputs(outputs, clusters):
+    """Cluster the outputs based on the number of right keypoints"""
+
+    # Check for "no right keypoints" condition
+    if clusters == 0:
+        clusters = max(1, round(outputs.shape[0] / 2))
+
+    assert outputs.shape[0] % clusters == 0, "Unexpected number of inputs"
+    outputs = outputs.view(-1, clusters, outputs.shape[1])
+    return outputs
+
+
+def filter_outputs(outputs):
+    """Extract a single output for each left keypoint"""
+
+    # Max of auxiliary task
+    val = outputs[:, :, -1]
+    best_val, _ = val.max(dim=1, keepdim=True)
+    mask = val >= best_val
+    output = outputs[mask]  # broadcasting happens only if 3rd dim not present
+    return output, mask
+
+
+def extract_outputs_mono(outputs, tasks=None):
+    """
+    Extract the outputs for single di
+    Inputs:
+        tensor (m, 10) or (m,9) if monoloco
+    Outputs:
+         - if tasks are provided return ordered list of raw tensors
+         - else return a dictionary with processed outputs
+    """
+    dic_out = {'xyz': outputs[:, 0:3], 'zb': outputs[:, 2:4],
+               'h': outputs[:, 4:5], 'w': outputs[:, 5:6], 'l': outputs[:, 6:7], 'ori': outputs[:, 7:9]}
+
+    # Multi-task training
+    if tasks is not None:
+        assert isinstance(tasks, tuple), "tasks need to be a tuple"
+        return [dic_out[task] for task in tasks]
+
+    # Preprocess the tensor
+    bi = unnormalize_bi(dic_out['zb'])
+
+    dic_out = {key: el.detach().cpu() for key, el in dic_out.items()}
+    dd = torch.norm(dic_out['xyz'], p=2, dim=1).view(-1, 1)
+    dic_out['xyzd'] = torch.cat((dic_out['xyz'], dd), dim=1)
+
+    dic_out['d'], dic_out['bi'] = dd, bi
+
+    yaw_pred = torch.atan2(dic_out['ori'][:, 0:1], dic_out['ori'][:, 1:2])
+    yaw_orig = back_correct_angles(yaw_pred, dic_out['xyzd'][:, 0:3])
+
+    dic_out['yaw'] = (yaw_pred, yaw_orig)  # alpha, ry
+    return dic_out

monoloco/predict.py

@@ -1,122 +1,286 @@
+# pylint: disable=too-many-statements, too-many-branches, undefined-loop-variable
+
+"""
+Adapted from https://github.com/openpifpaf/openpifpaf/blob/main/openpifpaf/predict.py,
+which is: 'Copyright 2019-2021 by Sven Kreiss and contributors. All rights reserved.'
+and licensed under GNU AGPLv3
+"""
 
 import os
+import glob
 import json
+import copy
+import logging
+from collections import defaultdict
+
 
 import torch
-from PIL import Image
+import PIL
-from openpifpaf import show
+import openpifpaf
-
+import openpifpaf.datasets as datasets
+from openpifpaf import decoder, network, visualizer, show, logger
+try:
+    import gdown
+    DOWNLOAD = copy.copy(gdown.download)
+except ImportError:
+    DOWNLOAD = None
 from .visuals.printer import Printer
-from .network import PifPaf, ImageList, MonoLoco
+from .network import Loco
 from .network.process import factory_for_gt, preprocess_pifpaf
+from .activity import show_activities
+
+LOG = logging.getLogger(__name__)
+
+OPENPIFPAF_MODEL = 'https://drive.google.com/uc?id=1b408ockhh29OLAED8Tysd2yGZOo0N_SQ'
+MONOLOCO_MODEL_KI = 'https://drive.google.com/uc?id=1krkB8J9JhgQp4xppmDu-YBRUxZvOs96r'
+MONOLOCO_MODEL_NU = 'https://drive.google.com/uc?id=1BKZWJ1rmkg5AF9rmBEfxF1r8s8APwcyC'
+MONSTEREO_MODEL = 'https://drive.google.com/uc?id=1xztN07dmp2e_nHI6Lcn103SAzt-Ntg49'
+
+
+def get_torch_checkpoints_dir():
+    if hasattr(torch, 'hub') and hasattr(torch.hub, 'get_dir'):
+        # new in pytorch 1.6.0
+        base_dir = torch.hub.get_dir()
+    elif os.getenv('TORCH_HOME'):
+        base_dir = os.getenv('TORCH_HOME')
+    elif os.getenv('XDG_CACHE_HOME'):
+        base_dir = os.path.join(os.getenv('XDG_CACHE_HOME'), 'torch')
+    else:
+        base_dir = os.path.expanduser(os.path.join('~', '.cache', 'torch'))
+    return os.path.join(base_dir, 'checkpoints')
+
+
+def download_checkpoints(args):
+    torch_dir = get_torch_checkpoints_dir()
+    os.makedirs(torch_dir, exist_ok=True)
+    if args.checkpoint is None:
+        os.makedirs(torch_dir, exist_ok=True)
+        pifpaf_model = os.path.join(torch_dir, 'shufflenetv2k30-201104-224654-cocokp-d75ed641.pkl')
+        print(pifpaf_model)
+    else:
+        pifpaf_model = args.checkpoint
+    dic_models = {'keypoints': pifpaf_model}
+    if not os.path.exists(pifpaf_model):
+        assert DOWNLOAD is not None, \
+            "pip install gdown to download a pifpaf model, or pass the model path as --checkpoint"
+        LOG.info('Downloading OpenPifPaf model in %s', torch_dir)
+        DOWNLOAD(OPENPIFPAF_MODEL, pifpaf_model, quiet=False)
+
+    if args.mode == 'keypoints':
+        return dic_models
+    if args.model is not None:
+        assert os.path.exists(args.model), "Model path not found"
+        dic_models[args.mode] = args.model
+        return dic_models
+    if args.mode == 'stereo':
+        assert not args.social_distance, "Social distance not supported in stereo modality"
+        path = MONSTEREO_MODEL
+        name = 'monstereo-201202-1212.pkl'
+    elif ('social_distance' in args.activities) or args.webcam:
+        path = MONOLOCO_MODEL_NU
+        name = 'monoloco_pp-201207-1350.pkl'
+    else:
+        path = MONOLOCO_MODEL_KI
+        name = 'monoloco_pp-201203-1424.pkl'
+
+    model = os.path.join(torch_dir, name)
+    print(name)
+    dic_models[args.mode] = model
+    if not os.path.exists(model):
+        os.makedirs(torch_dir, exist_ok=True)
+        assert DOWNLOAD is not None, \
+            "pip install gdown to download a monoloco model, or pass the model path as --model"
+        LOG.info('Downloading model in %s', torch_dir)
+        DOWNLOAD(path, model, quiet=False)
+    return dic_models
+
+
+def factory_from_args(args):
+
+    # Data
+    if args.glob:
+        args.images += glob.glob(args.glob)
+    if not args.images:
+        raise Exception("no image files given")
+
+    if args.path_gt is None:
+        args.show_all = True
+
+    # Models
+    dic_models = download_checkpoints(args)
+    args.checkpoint = dic_models['keypoints']
+
+    logger.configure(args, LOG)  # logger first
+
+    # Devices
+    args.device = torch.device('cpu')
+    args.pin_memory = False
+    if torch.cuda.is_available():
+        args.device = torch.device('cuda')
+        args.pin_memory = True
+    LOG.debug('neural network device: %s', args.device)
+
+    # Add visualization defaults
+    args.figure_width = 10
+    args.dpi_factor = 1.0
+
+    if args.mode == 'stereo':
+        args.batch_size = 2
+        args.images = sorted(args.images)
+    else:
+        args.batch_size = 1
+
+    if args.casr_std:
+        args.casr = 'std'
+    elif args.casr:
+        args.casr = 'nonstd'
+
+    # Patch for stereo images with batch_size = 2
+    if args.batch_size == 2 and not args.long_edge:
+        args.long_edge = 1238
+        LOG.info("Long-edge set to %i", args.long_edge)
+
+    # Make default pifpaf argument
+    args.force_complete_pose = True
+    LOG.info("Force complete pose is active")
+
+    # Configure
+    decoder.configure(args)
+    network.Factory.configure(args)
+    show.configure(args)
+    visualizer.configure(args)
+
+    return args, dic_models
 
 
 def predict(args):
 
     cnt = 0
+    assert args.mode in ('keypoints', 'mono', 'stereo')
+    args, dic_models = factory_from_args(args)
 
-    # load pifpaf and monoloco models
+    # Load Models
-    pifpaf = PifPaf(args)
+    if args.mode in ('mono', 'stereo'):
-    monoloco = MonoLoco(model=args.model, device=args.device, n_dropout=args.n_dropout, p_dropout=args.dropout)
+        net = Loco(
+            model=dic_models[args.mode],
+            mode=args.mode,
+            device=args.device,
+            n_dropout=args.n_dropout,
+            p_dropout=args.dropout,
+            casr=args.casr,
+            casr_model=args.casr_model)
+
+    # for openpifpaf predicitons
+    predictor = openpifpaf.Predictor(checkpoint=args.checkpoint)
 
     # data
-    data = ImageList(args.images, scale=args.scale)
+    data = datasets.ImageList(args.images, preprocess=predictor.preprocess)
+    if args.mode == 'stereo':
+        assert len(
+            data.image_paths) % 2 == 0, "Odd number of images in a stereo setting"
+
     data_loader = torch.utils.data.DataLoader(
-        data, batch_size=1, shuffle=False,
+        data, batch_size=args.batch_size, shuffle=False,
-        pin_memory=args.pin_memory, num_workers=args.loader_workers)
+        pin_memory=False, collate_fn=datasets.collate_images_anns_meta)
 
-    for idx, (image_paths, image_tensors, processed_images_cpu) in enumerate(data_loader):
+    for batch_i, (_, _, meta_batch) in enumerate(data_loader):
-        images = image_tensors.permute(0, 2, 3, 1)
 
-        processed_images = processed_images_cpu.to(args.device, non_blocking=True)
+        # unbatch (only for MonStereo)
-        fields_batch = pifpaf.fields(processed_images)
+        for idx, (preds, _, meta) in enumerate(predictor.dataset(data)):
+            LOG.info('batch %d: %s', batch_i, meta['file_name'])
 
-        # unbatch
+            # Load image and collect pifpaf results
-        for image_path, image, processed_image_cpu, fields in zip(
+            if idx == 0:
-                image_paths, images, processed_images_cpu, fields_batch):
+                with open(meta_batch[0]['file_name'], 'rb') as f:
+                    cpu_image = PIL.Image.open(f).convert('RGB')
+                pifpaf_outs = {
+                    'pred': preds,
+                    'left': [ann.json_data() for ann in preds],
+                    'image': cpu_image}
 
+                # Set output image name
                 if args.output_directory is None:
-                output_path = image_path
+                    splits = os.path.split(meta['file_name'])
+                    output_path = os.path.join(splits[0], 'out_' + splits[1])
                 else:
-                file_name = os.path.basename(image_path)
+                    file_name = os.path.basename(meta['file_name'])
-                output_path = os.path.join(args.output_directory, file_name)
+                    output_path = os.path.join(
-            print('image', idx, image_path, output_path)
+                        args.output_directory, 'out_' + file_name)
 
-            keypoint_sets, scores, pifpaf_out = pifpaf.forward(image, processed_image_cpu, fields)
+                im_name = os.path.basename(meta['file_name'])
-            pifpaf_outputs = [keypoint_sets, scores, pifpaf_out]  # keypoints_sets and scores for pifpaf printing
+                print(f'{batch_i} image {im_name} saved as {output_path}')
-            images_outputs = [image]  # List of 1 or 2 elements with pifpaf tensor (resized) and monoloco original image
 
-            if 'monoloco' in args.networks:
+            # Only for MonStereo
-                im_size = (float(image.size()[1] / args.scale),
+            else:
-                           float(image.size()[0] / args.scale))  # Width, Height (original)
+                pifpaf_outs['right'] = [ann.json_data() for ann in preds]
 
-                # Extract calibration matrix and ground truth file if present
+        # 3D Predictions
-                with open(image_path, 'rb') as f:
+        if args.mode != 'keypoints':
-                    pil_image = Image.open(f).convert('RGB')
+            im_size = (cpu_image.size[0], cpu_image.size[1])  # Original
-                    images_outputs.append(pil_image)
+            kk, dic_gt = factory_for_gt(
-
+                im_size, focal_length=args.focal, name=im_name, path_gt=args.path_gt)
-                im_name = os.path.basename(image_path)
-
-                kk, dic_gt = factory_for_gt(im_size, name=im_name, path_gt=args.path_gt)
 
             # Preprocess pifpaf outputs and run monoloco
-                boxes, keypoints = preprocess_pifpaf(pifpaf_out, im_size)
+            boxes, keypoints = preprocess_pifpaf(
-                outputs, varss = monoloco.forward(keypoints, kk)
+                pifpaf_outs['left'], im_size, enlarge_boxes=False)
-                dic_out = monoloco.post_process(outputs, varss, boxes, keypoints, kk, dic_gt)
+
+            if args.mode == 'mono':
+                LOG.info("Prediction with MonoLoco++")
+                dic_out = net.forward(keypoints, kk)
+                dic_out = net.post_process(
+                    dic_out, boxes, keypoints, kk, dic_gt)
+                if 'social_distance' in args.activities:
+                    dic_out = net.social_distance(dic_out, args)
+                if 'raise_hand' in args.activities:
+                    dic_out = net.raising_hand(dic_out, keypoints)
+                if 'using_phone' in args.activities:
+                    dic_out = net.using_phone(dic_out, keypoints)
+                if 'is_turning' in args.activities:
+                    dic_out = net.turning_forward(dic_out, keypoints)
+            else:
+                LOG.info("Prediction with MonStereo")
+                _, keypoints_r = preprocess_pifpaf(pifpaf_outs['right'], im_size)
+                dic_out = net.forward(keypoints, kk, keypoints_r=keypoints_r)
+                dic_out = net.post_process(
+                    dic_out, boxes, keypoints, kk, dic_gt)
 
         else:
-                dic_out = None
+            dic_out = defaultdict(list)
             kk = None
 
-            factory_outputs(args, images_outputs, output_path, pifpaf_outputs, dic_out=dic_out, kk=kk)
+        # Outputs
-            print('Image {}\n'.format(cnt) + '-' * 120)
+        factory_outputs(args, pifpaf_outs, dic_out, output_path, kk=kk)
+        print(f'Image {cnt}\n' + '-' * 120)
         cnt += 1
 
 
-def factory_outputs(args, images_outputs, output_path, pifpaf_outputs, dic_out=None, kk=None):
+def factory_outputs(args, pifpaf_outs, dic_out, output_path, kk=None):
     """Output json files or images according to the choice"""
 
-    # Save json file
+    # Verify conflicting options
-    if 'pifpaf' in args.networks:
+    if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
-        keypoint_sets, scores, pifpaf_out = pifpaf_outputs[:]
+        assert args.mode != 'keypoints', "for keypoints please use pifpaf original arguments"
+    else:
+        assert 'json' in args.output_types or args.mode == 'keypoints', \
+            "No output saved, please select one among front, bird, multi, json, or pifpaf arguments"
+    if 'social_distance' in args.activities:
+        assert args.mode == 'mono', "Social distancing only works with monocular network"
 
-        # Visualizer
+    if args.mode == 'keypoints':
-        keypoint_painter = show.KeypointPainter(show_box=False)
+        annotation_painter = openpifpaf.show.AnnotationPainter()
-        skeleton_painter = show.KeypointPainter(show_box=False, color_connections=True,
+        with openpifpaf.show.image_canvas(pifpaf_outs['image'], output_path) as ax:
-                                                markersize=1, linewidth=4)
+            annotation_painter.annotations(ax, pifpaf_outs['pred'])
+        return
 
-        if 'json' in args.output_types and keypoint_sets.size > 0:
+    if any((xx in args.output_types for xx in ['front', 'bird', 'multi'])):
-            with open(output_path + '.pifpaf.json', 'w') as f:
+        LOG.info(output_path)
-                json.dump(pifpaf_out, f)
+        if args.activities:
-
+            show_activities(
-        if 'keypoints' in args.output_types:
+                args, pifpaf_outs['image'], output_path, pifpaf_outs['left'], dic_out)
-            with show.image_canvas(images_outputs[0],
+        else:
-                                   output_path + '.keypoints.png',
+            printer = Printer(pifpaf_outs['image'], output_path, kk, args)
-                                   show=args.show,
+            figures, axes = printer.factory_axes(dic_out)
-                                   fig_width=args.figure_width,
+            printer.draw(figures, axes, pifpaf_outs['image'])
-                                   dpi_factor=args.dpi_factor) as ax:
-                keypoint_painter.keypoints(ax, keypoint_sets)
-
-        if 'skeleton' in args.output_types:
-            with show.image_canvas(images_outputs[0],
-                                   output_path + '.skeleton.png',
-                                   show=args.show,
-                                   fig_width=args.figure_width,
-                                   dpi_factor=args.dpi_factor) as ax:
-                skeleton_painter.keypoints(ax, keypoint_sets, scores=scores)
-
-    if 'monoloco' in args.networks:
-        if any((xx in args.output_types for xx in ['front', 'bird', 'combined'])):
-            epistemic = False
-            if args.n_dropout > 0:
-                epistemic = True
-
-            if dic_out['boxes']:  # Only print in case of detections
-                printer = Printer(images_outputs[1], output_path, kk, output_types=args.output_types
-                                  , z_max=args.z_max, epistemic=epistemic)
-                figures, axes = printer.factory_axes()
-                printer.draw(figures, axes, dic_out, images_outputs[1], draw_box=args.draw_box,
-                             save=True, show=args.show)
 
     if 'json' in args.output_types:
         with open(os.path.join(output_path + '.monoloco.json'), 'w') as ff:

monoloco/prep/__init__.py

@@ -0,0 +1,3 @@
+
+from .preprocess_kitti import parse_ground_truth, factory_file
+from .preprocess_casr import create_dic

monoloco/prep/preprocess_casr.py

@@ -0,0 +1,100 @@
+import pickle
+import re
+import json
+import os
+import glob
+import datetime
+import numpy as np
+import torch
+
+from .. import __version__
+from ..network.process import preprocess_monoloco
+
+gt_path = 'data/casr/annotations/casr_annotation.pickle'
+res_path = '/scratch/izar/beauvill/casr/res_extended/casr*'
+
+def bb_intersection_over_union(boxA, boxB):
+    xA = max(boxA[0], boxB[0])
+    yA = max(boxA[1], boxB[1])
+    xB = min(boxA[2], boxB[2])
+    yB = min(boxA[3], boxB[3])
+    interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)
+    boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
+    boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)
+    iou = interArea / float(boxAArea + boxBArea - interArea)
+    return iou
+
+def match_bboxes(bbox_gt, bbox_pred):
+    n_true = bbox_gt.shape[0]
+    n_pred = bbox_pred.shape[0]
+
+    iou_matrix = np.zeros((n_true, n_pred))
+    for i in range(n_true):
+        for j in range(n_pred):
+            iou_matrix[i, j] = bb_intersection_over_union(bbox_gt[i,:], bbox_pred[j,:])
+
+    return np.argmax(iou_matrix)
+
+def standard_bbox(bbox):
+    return [bbox[0], bbox[1], bbox[0]+bbox[2], bbox[1]+bbox[3]]
+
+def load_gt():
+    return pickle.load(open(gt_path, 'rb'), encoding='latin1')
+
+def load_res(path=res_path):
+    mono = []
+    for folder in sorted(glob.glob(path), key=lambda x:float(re.findall(r"(\d+)",x)[0])):
+        data_list = []
+        for file in sorted(os.listdir(folder), key=lambda x:float(re.findall(r"(\d+)",x)[0])):
+            if 'json' in file:
+                json_path = os.path.join(folder, file)
+                json_data = json.load(open(json_path))
+                json_data['filename'] = json_path
+                data_list.append(json_data)
+        mono.append(data_list)
+    return mono
+
+def create_dic(dir_ann=res_path, std=False):
+    gt=load_gt()
+    res=load_res(dir_ann)
+    dic_jo = {
+        'train': dict(X=[], Y=[], names=[], kps=[]),
+        'val': dict(X=[], Y=[], names=[], kps=[]),
+        'version': __version__,
+    }
+    split = ['3', '4']
+    if std:
+        wrong = [6, 8, 9, 10, 11, 12, 14, 21, 40, 43, 55, 70, 76, 92, 109,
+                 110, 112, 113, 121, 123, 124, 127, 128, 134, 136, 139, 165, 173]
+        mode = 'std'
+    else:
+        wrong = []
+        mode = ''
+    for i in [x for x in range(len(res[:])) if x not in wrong]:
+        for j in [x for x in range(len(res[i][:])) if 'boxes' in res[i][x]]:
+            folder = gt[i][j]['video_folder']
+
+            phase = 'val'
+            if folder[7] in split:
+                phase = 'train'
+
+            gt_box = gt[i][j]['bbox_gt']
+
+            good_idx = match_bboxes(np.array([standard_bbox(gt_box)]), np.array(res[i][j]['boxes'])[:,:4])
+
+            keypoints = [res[i][j]['uv_kps'][good_idx]]
+
+            gt_turn = gt[i][j]['left_or_right']
+            if std and gt_turn == 3:
+                gt_turn = 2
+            inp = preprocess_monoloco(keypoints, torch.eye(3)).view(-1).tolist()
+            dic_jo[phase]['kps'].append(keypoints)
+            dic_jo[phase]['X'].append(inp)
+            dic_jo[phase]['Y'].append(gt_turn)
+            dic_jo[phase]['names'].append(folder+"_frame{}".format(j))
+
+    now_time = datetime.datetime.now().strftime("%Y%m%d-%H%M")[2:]
+    with open("data/casr/outputs/joints-casr-" + mode + "-right-" +
+              split[0] + split[1] + "-" + now_time + ".json", 'w') as file:
+        json.dump(dic_jo, file)
+    return dic_jo

monoloco/prep/preprocess_ki.py

@@ -1,132 +0,0 @@
-"""Preprocess annotations with KITTI ground-truth"""
-
-import os
-import glob
-import copy
-import logging
-from collections import defaultdict
-import json
-import datetime
-
-from .transforms import transform_keypoints
-from ..utils import get_calibration, split_training, parse_ground_truth, get_iou_matches, append_cluster
-from ..network.process import preprocess_pifpaf, preprocess_monoloco
-
-
-class PreprocessKitti:
-    """Prepare arrays with same format as nuScenes preprocessing but using ground truth txt files"""
-
-    logging.basicConfig(level=logging.INFO)
-    logger = logging.getLogger(__name__)
-
-    dic_jo = {'train': dict(X=[], Y=[], names=[], kps=[], boxes_3d=[], K=[],
-                            clst=defaultdict(lambda: defaultdict(list))),
-              'val': dict(X=[], Y=[], names=[], kps=[], boxes_3d=[], K=[],
-                          clst=defaultdict(lambda: defaultdict(list))),
-              'test': dict(X=[], Y=[], names=[], kps=[], boxes_3d=[], K=[],
-                           clst=defaultdict(lambda: defaultdict(list)))}
-    dic_names = defaultdict(lambda: defaultdict(list))
-
-    def __init__(self, dir_ann, iou_min):
-
-        self.dir_ann = dir_ann
-        self.iou_min = iou_min
-        self.dir_gt = os.path.join('data', 'kitti', 'gt')
-        self.names_gt = tuple(os.listdir(self.dir_gt))
-        self.dir_kk = os.path.join('data', 'kitti', 'calib')
-        self.list_gt = glob.glob(self.dir_gt + '/*.txt')
-        assert os.path.exists(self.dir_gt), "Ground truth dir does not exist"
-        assert os.path.exists(self.dir_ann), "Annotation dir does not exist"
-
-        now = datetime.datetime.now()
-        now_time = now.strftime("%Y%m%d-%H%M")[2:]
-        dir_out = os.path.join('data', 'arrays')
-        self.path_joints = os.path.join(dir_out, 'joints-kitti-' + now_time + '.json')
-        self.path_names = os.path.join(dir_out, 'names-kitti-' + now_time + '.json')
-        path_train = os.path.join('splits', 'kitti_train.txt')
-        path_val = os.path.join('splits', 'kitti_val.txt')
-        self.set_train, self.set_val = split_training(self.names_gt, path_train, path_val)
-
-    def run(self):
-        """Save json files"""
-
-        cnt_gt = cnt_files = cnt_files_ped = cnt_fnf = 0
-        dic_cnt = {'train': 0, 'val': 0, 'test': 0}
-
-        for name in self.names_gt:
-            path_gt = os.path.join(self.dir_gt, name)
-            basename, _ = os.path.splitext(name)
-
-            phase, flag = self._factory_phase(name)
-            if flag:
-                cnt_fnf += 1
-                continue
-
-            # Extract keypoints
-            path_txt = os.path.join(self.dir_kk, basename + '.txt')
-            p_left, _ = get_calibration(path_txt)
-            kk = p_left[0]
-
-            # Iterate over each line of the gt file and save box location and distances
-            boxes_gt, boxes_3d, dds_gt = parse_ground_truth(path_gt, category='all')[:3]
-
-            self.dic_names[basename + '.png']['boxes'] = copy.deepcopy(boxes_gt)
-            self.dic_names[basename + '.png']['dds'] = copy.deepcopy(dds_gt)
-            self.dic_names[basename + '.png']['K'] = copy.deepcopy(kk)
-            cnt_gt += len(boxes_gt)
-            cnt_files += 1
-            cnt_files_ped += min(len(boxes_gt), 1)  # if no boxes 0 else 1
-
-            # Find the annotations if exists
-            try:
-                with open(os.path.join(self.dir_ann, basename + '.png.pifpaf.json'), 'r') as f:
-                    annotations = json.load(f)
-                boxes, keypoints = preprocess_pifpaf(annotations, im_size=(1238, 374))
-                keypoints_hflip = transform_keypoints(keypoints, mode='flip')
-                inputs = preprocess_monoloco(keypoints, kk).tolist()
-                inputs_hflip = preprocess_monoloco(keypoints_hflip, kk).tolist()
-                all_keypoints = [keypoints, keypoints_hflip]
-                all_inputs = [inputs, inputs_hflip]
-
-            except FileNotFoundError:
-                boxes = []
-
-            # Match each set of keypoint with a ground truth
-            matches = get_iou_matches(boxes, boxes_gt, self.iou_min)
-            for (idx, idx_gt) in matches:
-                for nn, keypoints in enumerate(all_keypoints):
-                    inputs = all_inputs[nn]
-                    self.dic_jo[phase]['kps'].append(keypoints[idx])
-                    self.dic_jo[phase]['X'].append(inputs[idx])
-                    self.dic_jo[phase]['Y'].append([dds_gt[idx_gt]])  # Trick to make it (nn,1)
-                    self.dic_jo[phase]['boxes_3d'].append(boxes_3d[idx_gt])
-                    self.dic_jo[phase]['K'].append(kk)
-                    self.dic_jo[phase]['names'].append(name)  # One image name for each annotation
-                    append_cluster(self.dic_jo, phase, inputs[idx], dds_gt[idx_gt], keypoints[idx])
-                dic_cnt[phase] += 1
-
-        with open(self.path_joints, 'w') as file:
-            json.dump(self.dic_jo, file)
-        with open(os.path.join(self.path_names), 'w') as file:
-            json.dump(self.dic_names, file)
-        for phase in ['train', 'val', 'test']:
-            print("Saved {} annotations for phase {}"
-                  .format(dic_cnt[phase], phase))
-        print("Number of GT files: {}. Files with at least one pedestrian: {}.  Files not found: {}"
-              .format(cnt_files, cnt_files_ped, cnt_fnf))
-        print("Matched : {:.1f} % of the ground truth instances"
-              .format(100 * (dic_cnt['train'] + dic_cnt['val']) / cnt_gt))
-        print("\nOutput files:\n{}\n{}\n".format(self.path_names, self.path_joints))
-
-    def _factory_phase(self, name):
-        """Choose the phase"""
-
-        phase = None
-        flag = False
-        if name in self.set_train:
-            phase = 'train'
-        elif name in self.set_val:
-            phase = 'val'
-        else:
-            flag = True
-        return phase, flag

monoloco/prep/preprocess_kitti.py

@@ -0,0 +1,392 @@
+# pylint: disable=too-many-statements, too-many-branches, too-many-nested-blocks
+
+"""Preprocess annotations with KITTI ground-truth"""
+
+import os
+import glob
+import copy
+import math
+import logging
+from collections import defaultdict
+import json
+import warnings
+import datetime
+from PIL import Image
+
+import torch
+
+from .. import __version__
+from ..utils import split_training, get_iou_matches, append_cluster, get_calibration, open_annotations, \
+    extract_stereo_matches, make_new_directory, \
+    check_conditions, to_spherical, correct_angle
+from ..network.process import preprocess_pifpaf, preprocess_monoloco
+from .transforms import flip_inputs, flip_labels, height_augmentation
+
+
+class PreprocessKitti:
+    """Prepare arrays with same format as nuScenes preprocessing but using ground truth txt files"""
+
+    # KITTI Dataset files
+    dir_gt = os.path.join('data', 'kitti', 'gt')
+    dir_images = os.path.join('data', 'kitti', 'images')
+    dir_kk = os.path.join('data', 'kitti', 'calib')
+
+    # SOCIAL DISTANCING PARAMETERS
+    THRESHOLD_DIST = 2  # Threshold to check distance of people
+    RADII = (0.3, 0.5, 1)  # expected radii of the o-space
+    SOCIAL_DISTANCE = True
+
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+
+    dic_jo = {
+        'train': dict(X=[], Y=[], names=[], kps=[], K=[], clst=defaultdict(lambda: defaultdict(list))),
+        'val': dict(X=[], Y=[], names=[], kps=[], K=[], clst=defaultdict(lambda: defaultdict(list))),
+        'test': dict(X=[], Y=[], names=[], kps=[], K=[], clst=defaultdict(lambda: defaultdict(list))),
+        'version': __version__,
+    }
+    dic_names = defaultdict(lambda: defaultdict(list))
+    dic_std = defaultdict(lambda: defaultdict(list))
+    categories_gt = dict(train=['Pedestrian', 'Person_sitting'], val=['Pedestrian'])
+
+    def __init__(self, dir_ann, mode='mono', iou_min=0.3, sample=False):
+
+        self.dir_ann = dir_ann
+        self.mode = mode
+        self.iou_min = iou_min
+        self.sample = sample
+
+        assert os.path.isdir(self.dir_ann), "Annotation directory not found"
+        assert any(os.scandir(self.dir_ann)), "Annotation directory empty"
+        assert os.path.isdir(self.dir_gt), "Ground truth directory not found"
+        assert any(os.scandir(self.dir_gt)), "Ground-truth directory empty"
+        if self.mode == 'stereo':
+            assert os.path.isdir(self.dir_ann + '_right'), "Annotation directory for right images not found"
+            assert any(os.scandir(self.dir_ann + '_right')), "Annotation directory for right images empty"
+        elif not os.path.isdir(self.dir_ann + '_right') or not any(os.scandir(self.dir_ann + '_right')):
+            warnings.warn('Horizontal flipping not applied as annotation directory for right images not found/empty')
+        assert self.mode in ('mono', 'stereo'), "modality not recognized"
+
+        self.names_gt = tuple(os.listdir(self.dir_gt))
+        self.list_gt = glob.glob(self.dir_gt + '/*.txt')
+        now = datetime.datetime.now()
+        now_time = now.strftime("%Y%m%d-%H%M")[2:]
+        dir_out = os.path.join('data', 'arrays')
+        self.path_joints = os.path.join(dir_out, 'joints-kitti-' + self.mode + '-' + now_time + '.json')
+        self.path_names = os.path.join(dir_out, 'names-kitti-' + self.mode + '-' + now_time + '.json')
+        path_train = os.path.join('splits', 'kitti_train.txt')
+        path_val = os.path.join('splits', 'kitti_val.txt')
+        self.set_train, self.set_val = split_training(self.names_gt, path_train, path_val)
+        self.phase, self.name = None, None
+        self.stats = defaultdict(int)
+        self.stats_stereo = defaultdict(int)
+
+    def run(self):
+        # self.names_gt = ('002282.txt',)
+        for self.name in self.names_gt:
+            # Extract ground truth
+            path_gt = os.path.join(self.dir_gt, self.name)
+            basename, _ = os.path.splitext(self.name)
+            self.phase, file_not_found = self._factory_phase(self.name)
+            category = 'all' if self.phase == 'train' else 'pedestrian'
+            if file_not_found:
+                self.stats['fnf'] += 1
+                continue
+
+            boxes_gt, labels, _, _, _ = parse_ground_truth(path_gt, category=category, spherical=True)
+            self.stats['gt_' + self.phase] += len(boxes_gt)
+            self.stats['gt_files'] += 1
+            self.stats['gt_files_ped'] += min(len(boxes_gt), 1)  # if no boxes 0 else 1
+            self.dic_names[basename + '.png']['boxes'] = copy.deepcopy(boxes_gt)
+            self.dic_names[basename + '.png']['ys'] = copy.deepcopy(labels)
+
+            # Extract annotations
+            dic_boxes, dic_kps, dic_gt = self.parse_annotations(boxes_gt, labels, basename)
+            if dic_boxes is None:  # No annotations
+                continue
+            self.dic_names[basename + '.png']['K'] = copy.deepcopy(dic_gt['K'])
+            self.dic_jo[self.phase]['K'].append(dic_gt['K'])
+
+            # Match each set of keypoint with a ground truth
+            for ii, boxes_gt in enumerate(dic_boxes['gt']):
+                kps, kps_r = torch.tensor(dic_kps['left'][ii]), torch.tensor(dic_kps['right'][ii])
+                matches = get_iou_matches(dic_boxes['left'][ii], boxes_gt, self.iou_min)
+                self.stats['flipping_match'] += len(matches) if ii == 1 else 0
+                for (idx, idx_gt) in matches:
+                    cat_gt = dic_gt['labels'][ii][idx_gt][-1]
+                    if cat_gt not in self.categories_gt[self.phase]: # only for training as cyclists are also extracted
+                        continue
+                    kp = kps[idx:idx + 1]
+                    kk = dic_gt['K']
+                    label = dic_gt['labels'][ii][idx_gt][:-1]
+                    self.stats['match'] += 1
+                    assert len(label) == 10, 'dimensions of monocular label is wrong'
+
+                    if self.mode == 'mono':
+                        self._process_annotation_mono(kp, kk, label)
+                    else:
+                        self._process_annotation_stereo(kp, kk, label, kps_r)
+
+        with open(self.path_joints, 'w') as file:
+            json.dump(self.dic_jo, file)
+        with open(os.path.join(self.path_names), 'w') as file:
+            json.dump(self.dic_names, file)
+        self._cout()
+
+    def parse_annotations(self, boxes_gt, labels, basename):
+
+        path_im = os.path.join(self.dir_images, basename + '.png')
+        path_calib = os.path.join(self.dir_kk, basename + '.txt')
+        min_conf = 0 if self.phase == 'train' else 0.1
+
+        # Check image size
+        with Image.open(path_im) as im:
+            width, height = im.size
+
+        # Extract left keypoints
+        annotations, kk, _ = factory_file(path_calib, self.dir_ann, basename)
+        boxes, keypoints = preprocess_pifpaf(annotations, im_size=(width, height), min_conf=min_conf)
+        if not keypoints:
+            return None, None, None
+
+        # Stereo-based horizontal flipping for training (obtaining ground truth for right images)
+        self.stats['instances'] += len(keypoints)
+        annotations_r, _, _ = factory_file(path_calib, self.dir_ann, basename, ann_type='right')
+        boxes_r, keypoints_r = preprocess_pifpaf(annotations_r, im_size=(width, height), min_conf=min_conf)
+
+        if not keypoints_r:  # Duplicate the left one(s)
+            all_boxes_gt, all_labels = [boxes_gt], [labels]
+            boxes_r, keypoints_r = boxes[0:1].copy(), keypoints[0:1].copy()
+            all_boxes, all_keypoints = [boxes], [keypoints]
+            all_keypoints_r = [keypoints_r]
+
+        elif self.phase == 'train':
+            # GT)
+            boxes_gt_flip, ys_flip = flip_labels(boxes_gt, labels, im_w=width)
+            # New left
+            boxes_flip = flip_inputs(boxes_r, im_w=width, mode='box')
+            keypoints_flip = flip_inputs(keypoints_r, im_w=width)
+
+            # New right
+            keypoints_r_flip = flip_inputs(keypoints, im_w=width)
+
+            # combine the 2 modes
+            all_boxes_gt = [boxes_gt, boxes_gt_flip]
+            all_labels = [labels, ys_flip]
+            all_boxes = [boxes, boxes_flip]
+            all_keypoints = [keypoints, keypoints_flip]
+            all_keypoints_r = [keypoints_r, keypoints_r_flip]
+
+        else:
+            all_boxes_gt, all_labels = [boxes_gt], [labels]
+            all_boxes, all_keypoints = [boxes], [keypoints]
+            all_keypoints_r = [keypoints_r]
+
+        dic_boxes = dict(left=all_boxes, gt=all_boxes_gt)
+        dic_kps = dict(left=all_keypoints, right=all_keypoints_r)
+        dic_gt = dict(K=kk, labels=all_labels)
+        return dic_boxes, dic_kps, dic_gt
+
+    def _process_annotation_mono(self, kp, kk, label):
+        """For a single annotation, process all the labels and save them"""
+        kp = kp.tolist()
+        inp = preprocess_monoloco(kp, kk).view(-1).tolist()
+
+        # Save
+        self.dic_jo[self.phase]['kps'].append(kp)
+        self.dic_jo[self.phase]['X'].append(inp)
+        self.dic_jo[self.phase]['Y'].append(label)
+        self.dic_jo[self.phase]['names'].append(self.name)  # One image name for each annotation
+        append_cluster(self.dic_jo, self.phase, inp, label, kp)
+        self.stats['total_' + self.phase] += 1
+
+    def _process_annotation_stereo(self, kp, kk, label, kps_r):
+        """For a reference annotation, combine it with some (right) annotations and save it"""
+
+        zz = label[2]
+        stereo_matches, cnt_amb = extract_stereo_matches(kp, kps_r, zz,
+                                                         phase=self.phase,
+                                                         seed=self.stats_stereo['pair'])
+        self.stats_stereo['ambiguous'] += cnt_amb
+
+        for idx_r, s_match in stereo_matches:
+            label_s = label + [s_match]  # add flag to distinguish "true pairs and false pairs"
+            self.stats_stereo['true_pair'] += 1 if s_match > 0.9 else 0
+            self.stats_stereo['pair'] += 1  # before augmentation
+
+            # ---> Remove noise of very far instances for validation
+            # if (self.phase == 'val') and (label[3] >= 50):
+            #     continue
+
+            #  ---> Save only positives unless there is no positive (keep positive flip and augm)
+            # if num > 0 and s_match < 0.9:
+            #     continue
+
+            # Height augmentation
+            flag_aug = False
+            if self.phase == 'train' and 3 < label[2] < 30 and (s_match > 0.9 or self.stats_stereo['pair'] % 2 == 0):
+                flag_aug = True
+
+            # Remove height augmentation
+            # flag_aug = False
+
+            if flag_aug:
+                kps_aug, labels_aug = height_augmentation(kp, kps_r[idx_r:idx_r + 1], label_s,
+                                                          seed=self.stats_stereo['pair'])
+            else:
+                kps_aug = [(kp, kps_r[idx_r:idx_r + 1])]
+                labels_aug = [label_s]
+
+            for i, lab in enumerate(labels_aug):
+                assert len(lab) == 11, 'dimensions of stereo label is wrong'
+                self.stats_stereo['pair_aug'] += 1
+                (kp_aug, kp_aug_r) = kps_aug[i]
+                input_l = preprocess_monoloco(kp_aug, kk).view(-1)
+                input_r = preprocess_monoloco(kp_aug_r, kk).view(-1)
+                keypoint = torch.cat((kp_aug, kp_aug_r), dim=2).tolist()
+                inp = torch.cat((input_l, input_l - input_r)).tolist()
+                self.dic_jo[self.phase]['kps'].append(keypoint)
+                self.dic_jo[self.phase]['X'].append(inp)
+                self.dic_jo[self.phase]['Y'].append(lab)
+                self.dic_jo[self.phase]['names'].append(self.name)  # One image name for each annotation
+                append_cluster(self.dic_jo, self.phase, inp, lab, keypoint)
+                self.stats_stereo['total_' + self.phase] += 1  # including height augmentation
+
+    def _cout(self):
+        print('-' * 100)
+        print(f"Number of GT files: {self.stats['gt_files']} ")
+        print(f"Files with at least one pedestrian/cyclist: {self.stats['gt_files_ped']}")
+        print(f"Files not found: {self.stats['fnf']}")
+        print('-' * 100)
+        our = self.stats['match'] - self.stats['flipping_match']
+        gt = self.stats['gt_train'] + self.stats['gt_val']
+        print(f"Ground truth matches: {100 * our  / gt:.1f} for left images (train and val)")
+        print(f"Parsed instances: {self.stats['instances']}")
+        print(f"Ground truth instances: {gt}")
+        print(f"Matched instances: {our}")
+        print(f"Including horizontal flipping: {self.stats['match']}")
+
+        if self.mode == 'stereo':
+            print('-' * 100)
+            print(f"Ambiguous instances removed: {self.stats_stereo['ambiguous']}")
+            print(f"True pairs ratio: {100 * self.stats_stereo['true_pair'] / self.stats_stereo['pair']:.1f}% ")
+            print(f"Height augmentation pairs: {self.stats_stereo['pair_aug'] - self.stats_stereo['pair']} ")
+            print('-' * 100)
+        total_train = self.stats_stereo['total_train'] if self.mode == 'stereo' else self.stats['total_train']
+        total_val = self.stats_stereo['total_val'] if self.mode == 'stereo' else self.stats['total_val']
+        print(f"Total annotations for TRAINING: {total_train}")
+        print(f"Total annotations for VALIDATION: {total_val}")
+        print('-' * 100)
+        print(f"\nOutput files:\n{self.path_names}\n{self.path_joints}")
+        print('-' * 100)
+
+    def process_activity(self):
+        """Augment ground-truth with flag activity"""
+
+        from monoloco.activity import social_interactions  # pylint: disable=import-outside-toplevel
+        main_dir = os.path.join('data', 'kitti')
+        dir_gt = os.path.join(main_dir, 'gt')
+        dir_out = os.path.join(main_dir, 'gt_activity')
+        make_new_directory(dir_out)
+        cnt_tp, cnt_tn = 0, 0
+
+        # Extract validation images for evaluation
+        category = 'pedestrian'
+
+        for name in self.set_val:
+            # Read
+            path_gt = os.path.join(dir_gt, name)
+            _, ys, _, _, lines = parse_ground_truth(path_gt, category, spherical=False)
+            angles = [y[10] for y in ys]
+            dds = [y[4] for y in ys]
+            xz_centers = [[y[0], y[2]] for y in ys]
+
+            # Write
+            path_out = os.path.join(dir_out, name)
+            with open(path_out, "w+") as ff:
+                for idx, line in enumerate(lines):
+                    if social_interactions(idx, xz_centers, angles, dds,
+                                           n_samples=1,
+                                           threshold_dist=self.THRESHOLD_DIST,
+                                           radii=self.RADII,
+                                           social_distance=self.SOCIAL_DISTANCE):
+                        activity = '1'
+                        cnt_tp += 1
+                    else:
+                        activity = '0'
+                        cnt_tn += 1
+
+                    line_new = line[:-1] + ' ' + activity + line[-1]
+                    ff.write(line_new)
+
+        print(f'Written {len(self.set_val)} new files in {dir_out}')
+        print(f'Saved {cnt_tp} positive and {cnt_tn} negative annotations')
+
+    def _factory_phase(self, name):
+        """Choose the phase"""
+        phase = None
+        flag = False
+        if name in self.set_train:
+            phase = 'train'
+        elif name in self.set_val:
+            phase = 'val'
+        else:
+            flag = True
+        return phase, flag
+
+
+def parse_ground_truth(path_gt, category, spherical=False):
+    """Parse KITTI ground truth files"""
+
+    boxes_gt = []
+    labels = []
+    truncs_gt = []  # Float from 0 to 1
+    occs_gt = []  # Either 0,1,2,3 fully visible, partly occluded, largely occluded, unknown
+    lines = []
+
+    with open(path_gt, "r") as f_gt:
+        for line_gt in f_gt:
+            line = line_gt.split()
+            if not check_conditions(line_gt, category, method='gt'):
+                continue
+            truncs_gt.append(float(line[1]))
+            occs_gt.append(int(line[2]))
+            boxes_gt.append([float(x) for x in line[4:8]])
+            xyz = [float(x) for x in line[11:14]]
+            hwl = [float(x) for x in line[8:11]]
+            dd = float(math.sqrt(xyz[0] ** 2 + xyz[1] ** 2 + xyz[2] ** 2))
+            yaw = float(line[14])
+            assert - math.pi <= yaw <= math.pi
+            alpha = float(line[3])
+            sin, cos, yaw_corr = correct_angle(yaw, xyz)
+            assert min(abs(-yaw_corr - alpha), (abs(yaw_corr - alpha))) < 0.15, "more than 10 degrees of error"
+            if spherical:
+                rtp = to_spherical(xyz)
+                loc = rtp[1:3] + xyz[2:3] + rtp[0:1]  # [theta, psi, z, r]
+            else:
+                loc = xyz + [dd]
+            cat = line[0]  # 'Pedestrian', or 'Person_sitting' for people
+            output = loc + hwl + [sin, cos, yaw, cat]
+            labels.append(output)
+            lines.append(line_gt)
+    return boxes_gt, labels, truncs_gt, occs_gt, lines
+
+
+def factory_file(path_calib, dir_ann, basename, ann_type='left'):
+    """Choose the annotation and the calibration files"""
+
+    assert ann_type in ('left', 'right')
+    p_left, p_right = get_calibration(path_calib)
+
+    if ann_type == 'left':
+        kk, tt = p_left[:]
+        path_ann = os.path.join(dir_ann, basename + '.png.predictions.json')
+
+    # The right folder is called <NameOfLeftFolder>_right
+    else:
+        kk, tt = p_right[:]
+        path_ann = os.path.join(dir_ann + '_right', basename + '.png.predictions.json')
+
+    annotations = open_annotations(path_ann)
+
+    return annotations, kk, tt

monoloco/prep/preprocess_nu.py

@@ -1,9 +1,14 @@
+# pylint: disable=too-many-statements, import-error
+
+
 """Extract joints annotations and match with nuScenes ground truths
 """
 
 import os
 import sys
 import time
+import math
+import copy
 import json
 import logging
 from collections import defaultdict
@@ -12,15 +17,21 @@ import datetime
 import numpy as np
 from nuscenes.nuscenes import NuScenes
 from nuscenes.utils import splits
+from pyquaternion import Quaternion
 
-from ..utils import get_iou_matches, append_cluster, select_categories, project_3d
+from ..utils import get_iou_matches, append_cluster, select_categories, project_3d, correct_angle, normalize_hwl, \
+    to_spherical
 from ..network.process import preprocess_pifpaf, preprocess_monoloco
 
 
 class PreprocessNuscenes:
-    """
+    """Preprocess Nuscenes dataset"""
-    Preprocess Nuscenes dataset
+    AV_W = 0.68
-    """
+    AV_L = 0.75
+    AV_H = 1.72
+    WLH_STD = 0.1
+    social = False
+
     CAMERAS = ('CAM_FRONT', 'CAM_FRONT_LEFT', 'CAM_FRONT_RIGHT', 'CAM_BACK', 'CAM_BACK_LEFT', 'CAM_BACK_RIGHT')
     dic_jo = {'train': dict(X=[], Y=[], names=[], kps=[], boxes_3d=[], K=[],
                             clst=defaultdict(lambda: defaultdict(list))),
@@ -76,17 +87,25 @@ class PreprocessNuscenes:
             while not current_token == "":
                 sample_dic = self.nusc.get('sample', current_token)
                 cnt_samples += 1
-
+                # if (cnt_samples % 4 == 0) and (cnt_ann < 3000):
                 # Extract all the sample_data tokens for each sample
                 for cam in self.CAMERAS:
                     sd_token = sample_dic['data'][cam]
                     cnt_sd += 1
 
                     # Extract all the annotations of the person
-                    name, boxes_gt, boxes_3d, dds, kk = self.extract_from_token(sd_token)
+                    path_im, boxes_obj, kk = self.nusc.get_sample_data(sd_token, box_vis_level=1)  # At least one corner
+                    boxes_gt, boxes_3d, ys = extract_ground_truth(boxes_obj, kk)
+                    kk = kk.tolist()
+                    name = os.path.basename(path_im)
+                    basename, _ = os.path.splitext(name)
+
+                    self.dic_names[basename + '.jpg']['boxes'] = copy.deepcopy(boxes_gt)
+                    self.dic_names[basename + '.jpg']['ys'] = copy.deepcopy(ys)
+                    self.dic_names[basename + '.jpg']['K'] = copy.deepcopy(kk)
 
                     # Run IoU with pifpaf detections and save
-                    path_pif = os.path.join(self.dir_ann, name + '.pifpaf.json')
+                    path_pif = os.path.join(self.dir_ann, name + '.predictions.json')
                     exists = os.path.isfile(path_pif)
 
                     if exists:
@@ -95,22 +114,21 @@ class PreprocessNuscenes:
                             boxes, keypoints = preprocess_pifpaf(annotations, im_size=(1600, 900))
                     else:
                         continue
-
                     if keypoints:
-                        inputs = preprocess_monoloco(keypoints, kk).tolist()
-
                         matches = get_iou_matches(boxes, boxes_gt, self.iou_min)
                         for (idx, idx_gt) in matches:
-                            self.dic_jo[phase]['kps'].append(keypoints[idx])
+                            keypoint = keypoints[idx:idx + 1]
-                            self.dic_jo[phase]['X'].append(inputs[idx])
+                            inp = preprocess_monoloco(keypoint, kk).view(-1).tolist()
-                            self.dic_jo[phase]['Y'].append([dds[idx_gt]])  # Trick to make it (nn,1)
+                            lab = ys[idx_gt]
+                            lab = normalize_hwl(lab)
+                            self.dic_jo[phase]['kps'].append(keypoint)
+                            self.dic_jo[phase]['X'].append(inp)
+                            self.dic_jo[phase]['Y'].append(lab)
                             self.dic_jo[phase]['names'].append(name)  # One image name for each annotation
                             self.dic_jo[phase]['boxes_3d'].append(boxes_3d[idx_gt])
-                            self.dic_jo[phase]['K'].append(kk)
+                            append_cluster(self.dic_jo, phase, inp, lab, keypoint)
-                            append_cluster(self.dic_jo, phase, inputs[idx], dds[idx_gt], keypoints[idx])
                             cnt_ann += 1
                             sys.stdout.write('\r' + 'Saved annotations {}'.format(cnt_ann) + '\t')
-
                 current_token = sample_dic['next']
 
         with open(os.path.join(self.path_joints), 'w') as f:
@@ -119,35 +137,49 @@ class PreprocessNuscenes:
             json.dump(self.dic_names, f)
         end = time.time()
 
+        # extract_box_average(self.dic_jo['train']['boxes_3d'])
         print("\nSaved {} annotations for {} samples in {} scenes. Total time: {:.1f} minutes"
               .format(cnt_ann, cnt_samples, cnt_scenes, (end-start)/60))
         print("\nOutput files:\n{}\n{}\n".format(self.path_names, self.path_joints))
 
-    def extract_from_token(self, sd_token):
+
+def extract_ground_truth(boxes_obj, kk, spherical=True):
 
     boxes_gt = []
-        dds = []
     boxes_3d = []
-        path_im, boxes_obj, kk = self.nusc.get_sample_data(sd_token, box_vis_level=1)  # At least one corner
+    ys = []
-        kk = kk.tolist()
+
-        name = os.path.basename(path_im)
     for box_obj in boxes_obj:
+        # Select category
         if box_obj.name[:6] != 'animal':
             general_name = box_obj.name.split('.')[0] + '.' + box_obj.name.split('.')[1]
         else:
             general_name = 'animal'
         if general_name in select_categories('all'):
-                box = project_3d(box_obj, kk)
-                dd = np.linalg.norm(box_obj.center)
-                boxes_gt.append(box)
-                dds.append(dd)
-                box_3d = box_obj.center.tolist() + box_obj.wlh.tolist()
-                boxes_3d.append(box_3d)
-                self.dic_names[name]['boxes'].append(box)
-                self.dic_names[name]['dds'].append(dd)
-                self.dic_names[name]['K'] = kk
 
-        return name, boxes_gt, boxes_3d, dds, kk
+            # Obtain 2D & 3D box
+            boxes_gt.append(project_3d(box_obj, kk))
+            boxes_3d.append(box_obj.center.tolist() + box_obj.wlh.tolist())
+
+            # Angle
+            yaw = quaternion_yaw(box_obj.orientation)
+            assert - math.pi <= yaw <= math.pi
+            sin, cos, _ = correct_angle(yaw, box_obj.center)
+            hwl = [float(box_obj.wlh[i]) for i in (2, 0, 1)]
+
+            # Spherical coordinates
+            xyz = list(box_obj.center)
+            dd = np.linalg.norm(box_obj.center)
+            if spherical:
+                rtp = to_spherical(xyz)
+                loc = rtp[1:3] + xyz[2:3] + rtp[0:1]  # [theta, psi, z, r]
+            else:
+                loc = xyz + [dd]
+
+            output = loc + hwl + [sin, cos, yaw]
+            ys.append(output)
+
+    return boxes_gt, boxes_3d, ys
 
 
 def factory(dataset, dir_nuscenes):
@@ -175,3 +207,59 @@ def factory(dataset, dir_nuscenes):
         split_train, split_val = split_scenes['train'], split_scenes['val']
 
     return nusc, scenes, split_train, split_val
+
+
+def quaternion_yaw(q: Quaternion, in_image_frame: bool = True) -> float:
+    if in_image_frame:
+        v = np.dot(q.rotation_matrix, np.array([1, 0, 0]))
+        yaw = -np.arctan2(v[2], v[0])
+    else:
+        v = np.dot(q.rotation_matrix, np.array([1, 0, 0]))
+        yaw = np.arctan2(v[1], v[0])
+    return float(yaw)
+
+
+def extract_box_average(boxes_3d):
+    boxes_np = np.array(boxes_3d)
+    means = np.mean(boxes_np[:, 3:], axis=0)
+    stds = np.std(boxes_np[:, 3:], axis=0)
+    print(means)
+    print(stds)
+
+
+def extract_social(inputs, ys, keypoints, idx, matches):
+    """Output a (padded) version with all the 5 neighbours
+    - Take the ground feet and the output z
+    - make relative to the person (as social LSTM)"""
+    all_inputs = []
+
+    # Find the lowest relative ground foot
+    ground_foot = np.max(np.array(inputs)[:, [31, 33]], axis=1)
+    rel_ground_foot = ground_foot - ground_foot[idx]
+    rel_ground_foot = rel_ground_foot.tolist()
+
+    # Order the people based on their distance
+    base = np.array([np.mean(np.array(keypoints[idx][0])), np.mean(np.array(keypoints[idx][1]))])
+    # delta_input = [abs((inp[31] + inp[33]) / 2 - base) for inp in inputs]
+    delta_input = [np.linalg.norm(base - np.array([np.mean(np.array(kp[0])), np.mean(np.array(kp[1]))]))
+                   for kp in keypoints]
+    sorted_indices = sorted(range(len(delta_input)), key=lambda k: delta_input[k])  # Return a list of sorted indices
+    all_inputs.extend(inputs[idx])
+
+    indices_idx = [idx for (idx, idx_gt) in matches]
+    for ii in range(1, 3):
+        try:
+            index = sorted_indices[ii]
+
+            # Extract the idx_gt corresponding to the input we are attaching if it exists
+            try:
+                idx_idx_gt = indices_idx.index(index)
+                idx_gt = matches[idx_idx_gt][1]
+                all_inputs.append(rel_ground_foot[index])  # Relative lower ground foot
+                all_inputs.append(float(ys[idx_gt][3]))  # Output Z
+            except ValueError:
+                all_inputs.extend([0.] * 2)
+        except IndexError:
+            all_inputs.extend([0.] * 2)
+    assert len(all_inputs) == 34 + 2 * 2
+    return all_inputs

monoloco/prep/transforms.py

@@ -1,28 +1,34 @@
 
+import math
+from copy import deepcopy
+
 import numpy as np
 
+from ..utils import correct_angle, to_cartesian, to_spherical
+
+BASELINE = 0.54
+BF = BASELINE * 721
 
 COCO_KEYPOINTS = [
-    'nose',            # 1
+    'nose',            # 0
-    'left_eye',        # 2
+    'left_eye',        # 1
-    'right_eye',       # 3
+    'right_eye',       # 2
-    'left_ear',        # 4
+    'left_ear',        # 3
-    'right_ear',       # 5
+    'right_ear',       # 4
-    'left_shoulder',   # 6
+    'left_shoulder',   # 5
-    'right_shoulder',  # 7
+    'right_shoulder',  # 6
-    'left_elbow',      # 8
+    'left_elbow',      # 7
-    'right_elbow',     # 9
+    'right_elbow',     # 8
-    'left_wrist',      # 10
+    'left_wrist',      # 9
-    'right_wrist',     # 11
+    'right_wrist',     # 10
-    'left_hip',        # 12
+    'left_hip',        # 11
-    'right_hip',       # 13
+    'right_hip',       # 12
-    'left_knee',       # 14
+    'left_knee',       # 13
-    'right_knee',      # 15
+    'right_knee',      # 14
-    'left_ankle',      # 16
+    'left_ankle',      # 15
-    'right_ankle',     # 17
+    'right_ankle',     # 16
 ]
 
-
 HFLIP = {
     'nose': 'nose',
     'left_eye': 'right_eye',
@@ -45,10 +51,92 @@ HFLIP = {
 
 
 def transform_keypoints(keypoints, mode):
-
+    """Egocentric horizontal flip"""
     assert mode == 'flip', "mode not recognized"
     kps = np.array(keypoints)
     dic_kps = {key: kps[:, :, idx] for idx, key in enumerate(COCO_KEYPOINTS)}
     kps_hflip = np.array([dic_kps[value] for key, value in HFLIP.items()])
     kps_hflip = np.transpose(kps_hflip, (1, 2, 0))
     return kps_hflip.tolist()
+
+
+def flip_inputs(keypoints, im_w, mode=None):
+    """Horizontal flip the keypoints or the boxes in the image"""
+    if mode == 'box':
+        boxes = deepcopy(keypoints)
+        for box in boxes:
+            temp = box[2]
+            box[2] = im_w - box[0]
+            box[0] = im_w - temp
+        return boxes
+
+    keypoints = np.array(keypoints)
+    keypoints[:, 0, :] = im_w - keypoints[:, 0, :]  # Shifted
+    kps_flip = transform_keypoints(keypoints, mode='flip')
+    return kps_flip
+
+
+def flip_labels(boxes_gt, labels, im_w):
+    """Correct x, d positions and angles after horizontal flipping"""
+    boxes_flip = deepcopy(boxes_gt)
+    labels_flip = deepcopy(labels)
+
+    for idx, label_flip in enumerate(labels_flip):
+
+        # Flip the box and account for disparity
+        disp = BF / label_flip[2]
+        temp = boxes_flip[idx][2]
+        boxes_flip[idx][2] = im_w - boxes_flip[idx][0] + disp
+        boxes_flip[idx][0] = im_w - temp + disp
+
+        # Flip X and D
+        rtp = label_flip[3:4] + label_flip[0:2]  # Originally t,p,z,r
+        xyz = to_cartesian(rtp)
+        xyz[0] = -xyz[0] + BASELINE  # x
+        rtp_r = to_spherical(xyz)
+        label_flip[3], label_flip[0], label_flip[1] = rtp_r[0], rtp_r[1], rtp_r[2]
+
+        # FLip and correct the angle
+        yaw = label_flip[9]
+        yaw_n = math.copysign(1, yaw) * (np.pi - abs(yaw))  # Horizontal flipping change of angle
+
+        sin, cos, _ = correct_angle(yaw_n, xyz)
+        label_flip[7], label_flip[8], label_flip[9] = sin, cos, yaw_n
+
+    return boxes_flip, labels_flip
+
+
+def height_augmentation(kps, kps_r, label_s, seed=0):
+    """
+    label_s: theta, psi, z, rho, wlh, sin, cos, s_match
+    """
+    n_labels = 3 if label_s[-1] > 0.9 else 1
+    height_min = 1.2
+    height_max = 2
+    av_height = 1.71
+    kps_aug = [[kps.clone(), kps_r.clone()] for _ in range(n_labels+1)]
+    labels_aug = [label_s.copy() for _ in range(n_labels+1)]  # Maintain the original
+    np.random.seed(seed)
+    heights = np.random.uniform(height_min, height_max, n_labels)  # 3 samples
+    zzs = heights * label_s[2] / av_height
+    disp = BF / label_s[2]
+
+    rtp = label_s[3:4] + label_s[0:2]  # Originally t,p,z,r
+    xyz = to_cartesian(rtp)
+
+    for i in range(n_labels):
+
+        if zzs[i] < 2:
+            continue
+        # Update keypoints
+        disp_new = BF / zzs[i]
+        delta_disp = disp - disp_new
+        kps_aug[i][1][0, 0, :] = kps_aug[i][1][0, 0, :] + delta_disp
+
+        # Update labels
+        labels_aug[i][2] = zzs[i]
+        xyz[2] = zzs[i]
+        rho = np.linalg.norm(xyz)
+        labels_aug[i][3] = rho
+
+    return kps_aug, labels_aug

monoloco/run.py

@@ -1,8 +1,8 @@
-# pylint: disable=too-many-branches, too-many-statements
+# pylint: disable=too-many-branches, too-many-statements, import-outside-toplevel
+
 import argparse
 
-from openpifpaf.network import nets
+from openpifpaf import decoder, network, visualizer, show, logger
-from openpifpaf import decoder
 
 
 def cli():
@@ -15,75 +15,122 @@ def cli():
     training_parser = subparsers.add_parser("train")
     eval_parser = subparsers.add_parser("eval")
 
-    # Preprocess input data
-    prep_parser.add_argument('--dir_ann', help='directory of annotations of 2d joints', required=True)
-    prep_parser.add_argument('--dataset',
-                             help='datasets to preprocess: nuscenes, nuscenes_teaser, nuscenes_mini, kitti',
-                             default='nuscenes')
-    prep_parser.add_argument('--dir_nuscenes', help='directory of nuscenes devkit', default='data/nuscenes/')
-    prep_parser.add_argument('--iou_min', help='minimum iou to match ground truth', type=float, default=0.3)
-
     # Predict (2D pose and/or 3D location from images)
-    # General
-    predict_parser.add_argument('--networks', nargs='+', help='Run pifpaf and/or monoloco', default=['monoloco'])
     predict_parser.add_argument('images', nargs='*', help='input images')
     predict_parser.add_argument('--glob', help='glob expression for input images (for many images)')
+    predict_parser.add_argument('--checkpoint', help='pifpaf model')
     predict_parser.add_argument('-o', '--output-directory', help='Output directory')
-    predict_parser.add_argument('--output_types', nargs='+', default=['json'],
+    predict_parser.add_argument('--output_types', nargs='+', default=['multi'],
                                 help='what to output: json keypoints skeleton for Pifpaf'
-                                     'json bird front combined for Monoloco')
+                                     'json bird front or multi for MonStereo')
-    predict_parser.add_argument('--show', help='to show images', action='store_true')
+    predict_parser.add_argument('--no_save', help='to show images', action='store_true')
+    predict_parser.add_argument('--hide_distance', help='to not show the absolute distance of people from the camera',
+                                default=False, action='store_true')
+    predict_parser.add_argument('--dpi', help='image resolution', type=int, default=150)
+    predict_parser.add_argument('--long-edge', default=None, type=int,
+                                help='rescale the long side of the image (aspect ratio maintained)')
+    predict_parser.add_argument('--white-overlay',
+                                nargs='?', default=False, const=0.8, type=float,
+                                help='increase contrast to annotations by making image whiter')
+    predict_parser.add_argument('--font-size', default=0, type=int, help='annotation font size')
+    predict_parser.add_argument('--monocolor-connections', default=False, action='store_true',
+                                help='use a single color per instance')
+    predict_parser.add_argument('--instance-threshold', type=float, default=None, help='threshold for entire instance')
+    predict_parser.add_argument('--seed-threshold', type=float, default=0.5, help='threshold for single seed')
+    predict_parser.add_argument('--disable-cuda', action='store_true', help='disable CUDA')
+    predict_parser.add_argument('--precise-rescaling', dest='fast_rescaling', default=True, action='store_false',
+                                help='use more exact image rescaling (requires scipy)')
+    predict_parser.add_argument('--decoder-workers', default=None, type=int,
+                                help='number of workers for pose decoding, 0 for windows')
 
-    # Pifpaf
+    decoder.cli(parser)
-    nets.cli(predict_parser)
+    logger.cli(parser)
-    decoder.cli(predict_parser, force_complete_pose=True, instance_threshold=0.15)
+    network.Factory.cli(parser)
-    predict_parser.add_argument('--scale', default=1.0, type=float, help='change the scale of the image to preprocess')
+    show.cli(parser)
+    visualizer.cli(parser)
 
     # Monoloco
-    predict_parser.add_argument('--model', help='path of MonoLoco model to load', required=True)
+    predict_parser.add_argument('--activities', nargs='+',
-    predict_parser.add_argument('--hidden_size', type=int, help='Number of hidden units in the model', default=512)
+                                choices=['raise_hand', 'social_distance', 'using_phone', 'is_turning'],
-    predict_parser.add_argument('--path_gt', help='path of json file with gt 3d localization',
+                                help='Choose activities to show: social_distance, raise_hand', default=[])
-                                default='data/arrays/names-kitti-190513-1754.json')
+    predict_parser.add_argument('--mode', help='keypoints, mono, stereo', default='mono')
-    predict_parser.add_argument('--transform', help='transformation for the pose', default='None')
+    predict_parser.add_argument('--model', help='path of MonoLoco/MonStereo model to load')
-    predict_parser.add_argument('--draw_box', help='to draw box in the images', action='store_true')
+    predict_parser.add_argument('--casr_model', help='path of casr model to load')
-    predict_parser.add_argument('--predict', help='whether to make prediction', action='store_true')
+    predict_parser.add_argument('--net', help='only to select older MonoLoco model, otherwise use --mode')
-    predict_parser.add_argument('--z_max', type=int, help='maximum meters distance for predictions', default=22)
+    predict_parser.add_argument('--path_gt', help='path of json file with gt 3d localization')
+                                #default='data/arrays/names-kitti-200615-1022.json')
+    predict_parser.add_argument('--z_max', type=int, help='maximum meters distance for predictions', default=100)
     predict_parser.add_argument('--n_dropout', type=int, help='Epistemic uncertainty evaluation', default=0)
     predict_parser.add_argument('--dropout', type=float, help='dropout parameter', default=0.2)
-    predict_parser.add_argument('--webcam', help='monoloco streaming', action='store_true')
+    predict_parser.add_argument('--show_all', help='only predict ground-truth matches or all', action='store_true')
+    predict_parser.add_argument('--casr', help='predict casr', action='store_true')
+    predict_parser.add_argument('--casr_std', help='predict casr with only standard gestures', action='store_true')
+    predict_parser.add_argument('--webcam', help='monstereo streaming', action='store_true')
+    predict_parser.add_argument('--camera', help='device to use for webcam streaming', type=int, default=0)
+    predict_parser.add_argument('--focal', help='focal length in mm for a sensor size of 7.2x5.4 mm. (nuScenes)',
+                                type=float, default=5.7)
+
+    # Social distancing and social interactions
+    predict_parser.add_argument('--threshold_prob', type=float, help='concordance for samples', default=0.25)
+    predict_parser.add_argument('--threshold_dist', type=float, help='min distance of people', default=2.5)
+    predict_parser.add_argument('--radii', type=tuple, help='o-space radii', default=(0.3, 0.5, 1))
+
+    # Preprocess input data
+    prep_parser.add_argument('--dir_ann', help='directory of annotations of 2d joints', required=True)
+    prep_parser.add_argument('--mode', help='mono, stereo', default='mono')
+    prep_parser.add_argument('--dataset',
+                             help='datasets to preprocess: nuscenes, nuscenes_teaser, nuscenes_mini, kitti',
+                             default='kitti')
+    prep_parser.add_argument('--dir_nuscenes', help='directory of nuscenes devkit', default='data/nuscenes/')
+    prep_parser.add_argument('--casr_std', help='prep casr with only standard gestures', action='store_true')
+    prep_parser.add_argument('--iou_min', help='minimum iou to match ground truth', type=float, default=0.3)
+    prep_parser.add_argument('--variance', help='new', action='store_true')
+    prep_parser.add_argument('--activity', help='new', action='store_true')
 
     # Training
-    training_parser.add_argument('--joints', help='Json file with input joints',
+    training_parser.add_argument('--joints', help='Json file with input joints', required=True)
-                                 default='data/arrays/joints-nuscenes_teaser-190513-1846.json')
+    training_parser.add_argument('--mode', help='mono, stereo, casr, casr_std', default='mono')
-    training_parser.add_argument('--save', help='whether to not save model and log file', action='store_false')
+    training_parser.add_argument('--out', help='output_path, e.g., data/outputs/test.pkl')
-    training_parser.add_argument('-e', '--epochs', type=int, help='number of epochs to train for', default=150)
+    training_parser.add_argument('-e', '--epochs', type=int, help='number of epochs to train for', default=500)
-    training_parser.add_argument('--bs', type=int, default=256, help='input batch size')
+    training_parser.add_argument('--bs', type=int, default=512, help='input batch size')
-    training_parser.add_argument('--baseline', help='whether to train using the baseline', action='store_true')
+    training_parser.add_argument('--monocular', help='whether to train monoloco', action='store_true')
     training_parser.add_argument('--dropout', type=float, help='dropout. Default no dropout', default=0.2)
     training_parser.add_argument('--lr', type=float, help='learning rate', default=0.002)
-    training_parser.add_argument('--sched_step', type=float, help='scheduler step time (epochs)', default=20)
+    training_parser.add_argument('--sched_step', type=float, help='scheduler step time (epochs)', default=30)
-    training_parser.add_argument('--sched_gamma', type=float, help='Scheduler multiplication every step', default=0.9)
+    training_parser.add_argument('--sched_gamma', type=float, help='Scheduler multiplication every step', default=0.98)
-    training_parser.add_argument('--hidden_size', type=int, help='Number of hidden units in the model', default=256)
+    training_parser.add_argument('--hidden_size', type=int, help='Number of hidden units in the model', default=1024)
     training_parser.add_argument('--n_stage', type=int, help='Number of stages in the model', default=3)
     training_parser.add_argument('--hyp', help='run hyperparameters tuning', action='store_true')
     training_parser.add_argument('--multiplier', type=int, help='Size of the grid of hyp search', default=1)
     training_parser.add_argument('--r_seed', type=int, help='specify the seed for training and hyp tuning', default=1)
+    training_parser.add_argument('--print_loss', help='print training and validation losses', action='store_true')
+    training_parser.add_argument('--auto_tune_mtl', help='whether to use uncertainty to autotune losses',
+                                 action='store_true')
+    training_parser.add_argument('--no_save', help='to not save model and log file', action='store_true')
 
     # Evaluation
+    eval_parser.add_argument('--mode', help='mono, stereo', default='mono')
     eval_parser.add_argument('--dataset', help='datasets to evaluate, kitti or nuscenes', default='kitti')
+    eval_parser.add_argument('--activity', help='evaluate activities', action='store_true')
     eval_parser.add_argument('--geometric', help='to evaluate geometric distance', action='store_true')
     eval_parser.add_argument('--generate', help='create txt files for KITTI evaluation', action='store_true')
-    eval_parser.add_argument('--dir_ann', help='directory of annotations of 2d joints (for KITTI evaluation')
+    eval_parser.add_argument('--dir_ann', help='directory of annotations of 2d joints (for KITTI evaluation)')
-    eval_parser.add_argument('--model', help='path of MonoLoco model to load', required=True)
+    eval_parser.add_argument('--model', help='path of MonoLoco model to load')
     eval_parser.add_argument('--joints', help='Json file with input joints to evaluate (for nuScenes evaluation)')
     eval_parser.add_argument('--n_dropout', type=int, help='Epistemic uncertainty evaluation', default=0)
     eval_parser.add_argument('--dropout', type=float, help='dropout. Default no dropout', default=0.2)
-    eval_parser.add_argument('--hidden_size', type=int, help='Number of hidden units in the model', default=256)
+    eval_parser.add_argument('--hidden_size', type=int, help='Number of hidden units in the model', default=1024)
     eval_parser.add_argument('--n_stage', type=int, help='Number of stages in the model', default=3)
     eval_parser.add_argument('--show', help='whether to show statistic graphs', action='store_true')
     eval_parser.add_argument('--save', help='whether to save statistic graphs', action='store_true')
     eval_parser.add_argument('--verbose', help='verbosity of statistics', action='store_true')
-    eval_parser.add_argument('--stereo', help='include stereo baseline results', action='store_true')
+    eval_parser.add_argument('--new', help='new', action='store_true')
+    eval_parser.add_argument('--variance', help='evaluate keypoints variance', action='store_true')
+
+    eval_parser.add_argument('--net', help='Choose network: monoloco, monoloco_p, monoloco_pp, monstereo')
+    eval_parser.add_argument('--baselines', help='whether to evaluate stereo baselines', action='store_true')
+    eval_parser.add_argument('--generate_official', help='whether to add empty txt files for official evaluation',
+                             action='store_true')
 
     args = parser.parse_args()
     return args
@@ -96,59 +143,77 @@ def main():
             from .visuals.webcam import webcam
             webcam(args)
         else:
-            from . import predict
+            from .predict import predict
-            predict.predict(args)
+            predict(args)
 
     elif args.command == 'prep':
         if 'nuscenes' in args.dataset:
             from .prep.preprocess_nu import PreprocessNuscenes
             prep = PreprocessNuscenes(args.dir_ann, args.dir_nuscenes, args.dataset, args.iou_min)
             prep.run()
-        if 'kitti' in args.dataset:
+        elif 'casr' in args.dataset:
-            from .prep.preprocess_ki import PreprocessKitti
+            from .prep import create_dic
-            prep = PreprocessKitti(args.dir_ann, args.iou_min)
+            create_dic(dir_ann=args.dir_ann, std=args.casr_std)
+        else:
+            from .prep.preprocess_kitti import PreprocessKitti
+            prep = PreprocessKitti(args.dir_ann, mode=args.mode, iou_min=args.iou_min)
+            if args.activity:
+                prep.process_activity()
+            else:
                 prep.run()
 
     elif args.command == 'train':
         from .train import HypTuning
         if args.hyp:
             hyp_tuning = HypTuning(joints=args.joints, epochs=args.epochs,
-                                   baseline=args.baseline, dropout=args.dropout,
+                                    monocular=args.monocular, dropout=args.dropout,
-                                   multiplier=args.multiplier, r_seed=args.r_seed)
+                                    multiplier=args.multiplier, r_seed=args.r_seed,
-            hyp_tuning.train()
+                                    mode=args.mode)
+            hyp_tuning.train(args)
         else:
             from .train import Trainer
-            training = Trainer(joints=args.joints, epochs=args.epochs, bs=args.bs,
+            training = Trainer(args)
-                               baseline=args.baseline, dropout=args.dropout, lr=args.lr, sched_step=args.sched_step,
-                               n_stage=args.n_stage, sched_gamma=args.sched_gamma, hidden_size=args.hidden_size,
-                               r_seed=args.r_seed, save=args.save)
-
             _ = training.train()
             _ = training.evaluate()
 
     elif args.command == 'eval':
-        if args.geometric:
+        if args.activity:
+            from .eval.eval_activity import ActivityEvaluator
+            evaluator = ActivityEvaluator(args)
+            if 'collective' in args.dataset:
+                evaluator.eval_collective()
+            else:
+                evaluator.eval_kitti()
+
+        elif args.geometric:
             assert args.joints, "joints argument not provided"
-            from .eval import geometric_baseline
+            from .eval.geom_baseline import geometric_baseline
             geometric_baseline(args.joints)
 
+        elif args.variance:
+            from .eval.eval_variance import joints_variance
+            joints_variance(args.joints, clusters=None, dic_ms=None)
+
+        else:
             if args.generate:
-            from .eval import GenerateKitti
+                from .eval.generate_kitti import GenerateKitti
-            kitti_txt = GenerateKitti(args.model, args.dir_ann, p_dropout=args.dropout, n_dropout=args.n_dropout,
+                kitti_txt = GenerateKitti(args)
-                                      stereo=args.stereo)
                 kitti_txt.run()
 
             if args.dataset == 'kitti':
                 from .eval import EvalKitti
-            kitti_eval = EvalKitti(verbose=args.verbose, stereo=args.stereo)
+                kitti_eval = EvalKitti(args)
                 kitti_eval.run()
-            kitti_eval.printer(show=args.show, save=args.save)
+                kitti_eval.printer()
 
-        if 'nuscenes' in args.dataset:
+            elif 'nuscenes' in args.dataset:
                 from .train import Trainer
-            training = Trainer(joints=args.joints)
+                training = Trainer(args)
                 _ = training.evaluate(load=True, model=args.model, debug=False)
 
+            else:
+                raise ValueError("Option not recognized")
+
     else:
         raise ValueError("Main subparser not recognized or not provided")
 

monoloco/train/datasets.py

@@ -5,6 +5,42 @@ import torch
 from torch.utils.data import Dataset
 
 
+class ActivityDataset(Dataset):
+    """
+    Dataloader for activity dataset
+    """
+
+    def __init__(self, joints, phase):
+        """
+        Load inputs and outputs from the pickles files from gt joints, mask joints or both
+        """
+        assert(phase in ['train', 'val', 'test'])
+
+        with open(joints, 'r') as f:
+            dic_jo = json.load(f)
+
+        # Define input and output for normal training and inference
+        self.inputs_all = torch.tensor(dic_jo[phase]['X'])
+        self.outputs_all = torch.tensor(dic_jo[phase]['Y']).view(-1, 1)
+        # self.kps_all = torch.tensor(dic_jo[phase]['kps'])
+
+    def __len__(self):
+        """
+        :return: number of samples (m)
+        """
+        return self.inputs_all.shape[0]
+
+    def __getitem__(self, idx):
+        """
+        Reading the tensors when required. E.g. Retrieving one element or one batch at a time
+        :param idx: corresponding to m
+        """
+        inputs = self.inputs_all[idx, :]
+        outputs = self.outputs_all[idx]
+        # kps = self.kps_all[idx, :]
+        return inputs, outputs
+
+
 class KeypointsDataset(Dataset):
     """
     Dataloader fro nuscenes or kitti datasets
@@ -21,12 +57,16 @@ class KeypointsDataset(Dataset):
 
         # Define input and output for normal training and inference
         self.inputs_all = torch.tensor(dic_jo[phase]['X'])
-        self.outputs_all = torch.tensor(dic_jo[phase]['Y']).view(-1, 1)
+        self.outputs_all = torch.tensor(dic_jo[phase]['Y'])
         self.names_all = dic_jo[phase]['names']
         self.kps_all = torch.tensor(dic_jo[phase]['kps'])
+        self.version = dic_jo['version']
 
         # Extract annotations divided in clusters
+        if 'clst' in dic_jo[phase]:
             self.dic_clst = dic_jo[phase]['clst']
+        else:
+            self.dic_clst = None
 
     def __len__(self):
         """
@@ -54,3 +94,6 @@ class KeypointsDataset(Dataset):
         count = len(self.dic_clst[clst]['Y'])
 
         return inputs, outputs, count
+
+    def get_version(self):
+        return self.version

monoloco/train/hyp_tuning.py

@@ -15,17 +15,17 @@ from .trainer import Trainer
 
 class HypTuning:
 
-    def __init__(self, joints, epochs, baseline, dropout, multiplier=1, r_seed=1):
+    def __init__(self, joints, epochs, monocular,
+                 dropout, multiplier=1, r_seed=1, mode=None):
         """
         Initialize directories, load the data and parameters for the training
         """
 
         # Initialize Directories
         self.joints = joints
-        self.baseline = baseline
+        self.monocular = monocular
         self.dropout = dropout
         self.num_epochs = epochs
-        self.baseline = baseline
         self.r_seed = r_seed
         dir_out = os.path.join('data', 'models')
         dir_logs = os.path.join('data', 'logs')
@@ -33,7 +33,10 @@ class HypTuning:
         if not os.path.exists(dir_logs):
             os.makedirs(dir_logs)
 
-        name_out = 'hyp-baseline-' if baseline else 'hyp-monoloco-'
+        name_out = 'hyp-monoloco-' if monocular else 'hyp-ms-'
+        if mode:
+            name_out = ('hyp-casr-' if mode == 'casr' else
+                        'hyp-casr_std-' if mode == 'casr_std' else name_out)
 
         self.path_log = os.path.join(dir_logs, name_out)
         self.path_model = os.path.join(dir_out, name_out)
@@ -46,23 +49,23 @@ class HypTuning:
         np.random.seed(r_seed)
         self.sched_gamma_list = [0.8, 0.9, 1, 0.8, 0.9, 1] * multiplier
         random.shuffle(self.sched_gamma_list)
-        self.sched_step = [10, 20, 30, 40, 50, 60] * multiplier
+        self.sched_step = [10, 20, 40, 60, 80, 100] * multiplier
         random.shuffle(self.sched_step)
-        self.bs_list = [64, 128, 256, 512, 1024, 2048] * multiplier
+        self.bs_list = [64, 128, 256, 512, 512, 1024] * multiplier
         random.shuffle(self.bs_list)
-        self.hidden_list = [256, 256, 256, 256, 256, 256] * multiplier
+        self.hidden_list = [512, 1024, 2048, 512, 1024, 2048] * multiplier
         random.shuffle(self.hidden_list)
         self.n_stage_list = [3, 3, 3, 3, 3, 3] * multiplier
         random.shuffle(self.n_stage_list)
         # Learning rate
-        aa = math.log(0.001, 10)
+        aa = math.log(0.0005, 10)
-        bb = math.log(0.03, 10)
+        bb = math.log(0.01, 10)
         log_lr_list = np.random.uniform(aa, bb, int(6 * multiplier)).tolist()
         self.lr_list = [10 ** xx for xx in log_lr_list]
         # plt.hist(self.lr_list, bins=50)
         # plt.show()
 
-    def train(self):
+    def train(self, args):
         """Train multiple times using log-space random search"""
 
         best_acc_val = 20
@@ -77,10 +80,7 @@ class HypTuning:
             hidden_size = self.hidden_list[idx]
             n_stage = self.n_stage_list[idx]
 
-            training = Trainer(joints=self.joints, epochs=self.num_epochs,
+            training = Trainer(args)
-                               bs=bs, baseline=self.baseline, dropout=self.dropout, lr=lr, sched_step=sched_step,
-                               sched_gamma=sched_gamma, hidden_size=hidden_size, n_stage=n_stage,
-                               save=False, print_loss=False, r_seed=self.r_seed)
 
             best_epoch = training.train()
             dic_err, model = training.evaluate()
@@ -92,12 +92,12 @@ class HypTuning:
                 dic_best['lr'] = lr
                 dic_best['joints'] = self.joints
                 dic_best['bs'] = bs
-                dic_best['baseline'] = self.baseline
+                dic_best['monocular'] = self.monocular
                 dic_best['sched_gamma'] = sched_gamma
                 dic_best['sched_step'] = sched_step
                 dic_best['hidden_size'] = hidden_size
                 dic_best['n_stage'] = n_stage
-                dic_best['acc_val'] = dic_err['val']['all']['mean']
+                dic_best['acc_val'] = dic_err['val']['all']['d']
                 dic_best['best_epoch'] = best_epoch
                 dic_best['random_seed'] = self.r_seed
                 # dic_best['acc_test'] = dic_err['test']['all']['mean']
@@ -124,7 +124,8 @@ class HypTuning:
         print()
         self.logger.info("Accuracy in each cluster:")
 
-        for key in dic_err_best['val']:
+        if args.mode in ['mono', 'stereo']:
-            self.logger.info("Val: error in cluster {} = {} ".format(key, dic_err_best['val'][key]['mean']))
+            for key in ('10', '20', '30', '>30', 'all'):
+                self.logger.info("Val: error in cluster {} = {} ".format(key, dic_err_best['val'][key]['d']))
         self.logger.info("Final accuracy Val: {:.2f}".format(dic_best['acc_val']))
         self.logger.info("\nSaved the model: {}".format(self.path_model))

monoloco/train/losses.py

@@ -0,0 +1,264 @@
+"""
+Adapted from https://github.com/openpifpaf,
+which is: 'Copyright 2019-2021 by Sven Kreiss and contributors. All rights reserved.'
+and licensed under GNU AGPLv3
+"""
+
+
+import math
+import torch
+import torch.nn as nn
+import numpy as np
+import matplotlib.pyplot as plt
+
+from ..network import extract_labels, extract_labels_aux, extract_labels_cyclist, extract_outputs
+
+
+class AutoTuneMultiTaskLoss(torch.nn.Module):
+    def __init__(self, losses_tr, losses_val, lambdas, tasks):
+        super().__init__()
+
+        assert all(l in (0.0, 1.0) for l in lambdas)
+        self.losses = torch.nn.ModuleList(losses_tr)
+        self.losses_val = losses_val
+        self.lambdas = lambdas
+        self.tasks = tasks
+        self.log_sigmas = torch.nn.Parameter(torch.zeros((len(lambdas),), dtype=torch.float32), requires_grad=True)
+
+    def forward(self, outputs, labels, phase='train'):
+
+        assert phase in ('train', 'val')
+        out = extract_outputs(outputs, tasks=self.tasks)
+        gt_out = extract_labels(labels, tasks=self.tasks)
+        loss_values = [lam * l(o, g) / (2.0 * (log_sigma.exp() ** 2))
+                       for lam, log_sigma, l, o, g in zip(self.lambdas, self.log_sigmas, self.losses, out, gt_out)]
+
+        auto_reg = [log_sigma for log_sigma in self.log_sigmas]  # pylint: disable=unnecessary-comprehension
+
+        loss = sum(loss_values) + sum(auto_reg)
+        if phase == 'val':
+            loss_values_val = [l(o, g) for l, o, g in zip(self.losses_val, out, gt_out)]
+            loss_values_val.extend([s.exp() for s in self.log_sigmas])
+            return loss, loss_values_val
+        return loss, loss_values
+
+
+class MultiTaskLoss(torch.nn.Module):
+    def __init__(self, losses_tr, losses_val, lambdas, tasks):
+        super().__init__()
+
+        self.losses = torch.nn.ModuleList(losses_tr)
+        self.losses_val = losses_val
+        self.lambdas = lambdas
+        self.tasks = tasks
+        if len(self.tasks) == 1 and self.tasks[0] == 'aux':
+            self.flag_aux = True
+            self.flag_cyclist = False
+        elif len(self.tasks) == 1 and self.tasks[0] == 'cyclist':
+            self.flag_cyclist = True
+            self.flag_aux = False
+        else:
+            self.flag_aux = False
+            self.flag_cyclist = False
+
+    def forward(self, outputs, labels, phase='train'):
+
+        assert phase in ('train', 'val')
+        out = extract_outputs(outputs, tasks=self.tasks)
+        if self.flag_aux:
+            gt_out = extract_labels_aux(labels, tasks=self.tasks)
+        elif self.flag_cyclist:
+            gt_out = extract_labels_cyclist(labels, tasks=self.tasks)
+        else:
+            gt_out = extract_labels(labels, tasks=self.tasks)
+        loss_values = [lam * l(o, g) for lam, l, o, g in zip(self.lambdas, self.losses, out, gt_out)]
+        loss = sum(loss_values)
+
+        if phase == 'val':
+            loss_values_val = [l(o, g) for l, o, g in zip(self.losses_val, out, gt_out)]
+            return loss, loss_values_val
+        return loss, loss_values
+
+
+class CompositeLoss(torch.nn.Module):
+
+    def __init__(self, tasks):
+        super().__init__()
+
+        self.tasks = tasks
+        self.multi_loss_tr = {task: (LaplacianLoss() if task == 'd'
+                                     else (nn.BCEWithLogitsLoss() if task in ('aux', )
+                                           else (nn.CrossEntropyLoss() if task == 'cyclist'
+                                           else nn.L1Loss()))) for task in tasks}
+
+        self.multi_loss_val = {}
+        for task in tasks:
+            if task == 'd':
+                loss = l1_loss_from_laplace
+            elif task == 'ori':
+                loss = angle_loss
+            elif task in ('aux', ):
+                loss = nn.BCEWithLogitsLoss()
+            elif task == 'cyclist':
+                loss = nn.CrossEntropyLoss()
+            else:
+                loss = nn.L1Loss()
+            self.multi_loss_val[task] = loss
+
+    def forward(self):
+        losses_tr = [self.multi_loss_tr[l] for l in self.tasks]
+        losses_val = [self.multi_loss_val[l] for l in self.tasks]
+        return losses_tr, losses_val
+
+
+class LaplacianLoss(torch.nn.Module):
+    """1D Gaussian with std depending on the absolute distance"""
+    def __init__(self, size_average=True, reduce=True, evaluate=False):
+        super().__init__()
+        self.size_average = size_average
+        self.reduce = reduce
+        self.evaluate = evaluate
+
+    def laplacian_1d(self, mu_si, xx):
+        """
+        1D Gaussian Loss. f(x | mu, sigma). The network outputs mu and sigma. X is the ground truth distance.
+        This supports backward().
+        Inspired by
+        https://github.com/naba89/RNN-Handwriting-Generation-Pytorch/blob/master/loss_functions.py
+
+        """
+        eps = 0.01  # To avoid 0/0 when no uncertainty
+        mu, si = mu_si[:, 0:1], mu_si[:, 1:2]
+        norm = 1 - mu / xx  # Relative
+        const = 2
+
+        term_a = torch.abs(norm) * torch.exp(-si) + eps
+        term_b = si
+        norm_bi = (np.mean(np.abs(norm.cpu().detach().numpy())), np.mean(torch.exp(si).cpu().detach().numpy()))
+
+        if self.evaluate:
+            return norm_bi
+        return term_a + term_b + const
+
+    def forward(self, outputs, targets):
+
+        values = self.laplacian_1d(outputs, targets)
+
+        if not self.reduce or self.evaluate:
+            return values
+        if self.size_average:
+            mean_values = torch.mean(values)
+            return mean_values
+        return torch.sum(values)
+
+
+class GaussianLoss(torch.nn.Module):
+    """1D Gaussian with std depending on the absolute distance
+    """
+    def __init__(self, device, size_average=True, reduce=True, evaluate=False):
+        super().__init__()
+        self.size_average = size_average
+        self.reduce = reduce
+        self.evaluate = evaluate
+        self.device = device
+
+    def gaussian_1d(self, mu_si, xx):
+        """
+        1D Gaussian Loss. f(x | mu, sigma). The network outputs mu and sigma. X is the ground truth distance.
+        This supports backward().
+        Inspired by
+        https://github.com/naba89/RNN-Handwriting-Generation-Pytorch/blob/master/loss_functions.py
+        """
+        mu, si = mu_si[:, 0:1], mu_si[:, 1:2]
+
+        min_si = torch.ones(si.size()).cuda(self.device) * 0.1
+        si = torch.max(min_si, si)
+        norm = xx - mu
+        term_a = (norm / si)**2 / 2
+        term_b = torch.log(si * math.sqrt(2 * math.pi))
+
+        norm_si = (np.mean(np.abs(norm.cpu().detach().numpy())), np.mean(si.cpu().detach().numpy()))
+
+        if self.evaluate:
+            return norm_si
+
+        return term_a + term_b
+
+    def forward(self, outputs, targets):
+
+        values = self.gaussian_1d(outputs, targets)
+
+        if not self.reduce or self.evaluate:
+            return values
+        if self.size_average:
+            mean_values = torch.mean(values)
+            return mean_values
+        return torch.sum(values)
+
+
+class CustomL1Loss(torch.nn.Module):
+    """
+    Experimental, not used.
+    L1 loss with more weight to errors at a shorter distance
+    It inherits from nn.module so it supports backward
+    """
+
+    def __init__(self, dic_norm, device, beta=1):
+        super().__init__()
+
+        self.dic_norm = dic_norm
+        self.device = device
+        self.beta = beta
+
+    @staticmethod
+    def compute_weights(xx, beta=1):
+        """
+        Return the appropriate weight depending on the distance and the hyperparameter chosen
+        alpha = 1 refers to the curve of A Photogrammetric Approach for Real-time...
+        It is made for unnormalized outputs (to be more understandable)
+        From 70 meters on every value is weighted the same (0.1**beta)
+        Alpha is optional value from Focal loss. Yet to be analyzed
+        """
+        # alpha = np.maximum(1, 10 ** (beta - 1))
+        alpha = 1
+        ww = np.maximum(0.1, 1 - xx / 78)**beta
+
+        return alpha * ww
+
+    def print_loss(self):
+        xx = np.linspace(0, 80, 100)
+        y1 = self.compute_weights(xx, beta=1)
+        y2 = self.compute_weights(xx, beta=2)
+        y3 = self.compute_weights(xx, beta=3)
+        plt.plot(xx, y1)
+        plt.plot(xx, y2)
+        plt.plot(xx, y3)
+        plt.xlabel("Distance [m]")
+        plt.ylabel("Loss function Weight")
+        plt.legend(("Beta = 1", "Beta = 2", "Beta = 3"))
+        plt.show()
+
+    def forward(self, output, target):
+
+        unnormalized_output = output.cpu().detach().numpy() * self.dic_norm['std']['Y'] + self.dic_norm['mean']['Y']
+        weights_np = self.compute_weights(unnormalized_output, self.beta)
+        weights = torch.from_numpy(weights_np).float().to(self.device)  # To make weights in the same cuda device
+        losses = torch.abs(output - target) * weights
+        loss = losses.mean()  # Mean over the batch
+        return loss
+
+
+def angle_loss(orient, gt_orient):
+    """Only for evaluation"""
+    angles = torch.atan2(orient[:, 0], orient[:, 1])
+    gt_angles = torch.atan2(gt_orient[:, 0], gt_orient[:, 1])
+    # assert all(angles < math.pi) & all(angles > - math.pi)
+    # assert all(gt_angles < math.pi) & all(gt_angles > - math.pi)
+    loss = torch.mean(torch.abs(angles - gt_angles)) * 180 / 3.14
+    return loss
+
+
+def l1_loss_from_laplace(out, gt_out):
+    """Only for evaluation"""
+    loss = torch.mean(torch.abs(out[:, 0:1] - gt_out))
+    return loss

monoloco/train/trainer.py

@@ -1,8 +1,10 @@
-
 # pylint: disable=too-many-statements
+
 """
-Training and evaluation of a neural network which predicts 3D localization and confidence intervals
+Training and evaluation of a neural network that, given 2D joints, estimates:
-given 2d joints
+- 3D localization and confidence intervals
+- Orientation
+- Bounding box dimensions
 """
 
 import copy
@@ -12,200 +14,195 @@ import logging
 from collections import defaultdict
 import sys
 import time
-import warnings
+from itertools import chain
+
+try:
+    import matplotlib.pyplot as plt
+except ImportError:
+    plt = None
 
-import matplotlib.pyplot as plt
 import torch
-import torch.nn as nn
 from torch.utils.data import DataLoader
 from torch.optim import lr_scheduler
 
+from .. import __version__
 from .datasets import KeypointsDataset
-from ..network import LaplacianLoss
+from .losses import CompositeLoss, MultiTaskLoss, AutoTuneMultiTaskLoss
-from ..network.process import unnormalize_bi
+from ..network import extract_outputs, extract_labels
-from ..network.architectures import LinearModel
+from ..network.architectures import LocoModel
 from ..utils import set_logger
 
 
 class Trainer:
-    def __init__(self, joints, epochs=100, bs=256, dropout=0.2, lr=0.002,
+    # Constants
-                 sched_step=20, sched_gamma=1, hidden_size=256, n_stage=3, r_seed=1, n_samples=100,
+    VAL_BS = 10000
-                 baseline=False, save=False, print_loss=False):
+
+    tasks = ('d', 'x', 'y', 'h', 'w', 'l', 'ori', 'aux')
+    val_task = 'd'
+    lambdas = (1, 1, 1, 1, 1, 1, 1, 1)
+    clusters = ['10', '20', '30', '40']
+    input_size = dict(mono=34, stereo=68, casr=34, casr_std=34)
+    output_size = dict(mono=9, stereo=10, casr=4, casr_std=3)
+    dir_figures = os.path.join('figures', 'losses')
+
+    def __init__(self, args):
         """
         Initialize directories, load the data and parameters for the training
         """
 
-        # Initialize directories and parameters
+        assert os.path.exists(args.joints), "Input file not found"
-        dir_out = os.path.join('data', 'models')
+        self.mode = args.mode
-        if not os.path.exists(dir_out):
+        self.joints = args.joints
-            warnings.warn("Warning: output directory not found, the model will not be saved")
+        self.num_epochs = args.epochs
-        dir_logs = os.path.join('data', 'logs')
+        self.no_save = args.no_save
-        if not os.path.exists(dir_logs):
+        self.print_loss = args.print_loss
-            warnings.warn("Warning: default logs directory not found")
+        self.lr = args.lr
-        assert os.path.exists(joints), "Input file not found"
+        self.sched_step = args.sched_step
+        self.sched_gamma = args.sched_gamma
+        self.hidden_size = args.hidden_size
+        self.n_stage = args.n_stage
+        self.r_seed = args.r_seed
+        self.auto_tune_mtl = args.auto_tune_mtl
+        self.is_casr = self.mode in ['casr', 'casr_std']
 
-        self.joints = joints
+        if self.is_casr:
-        self.num_epochs = epochs
+            self.tasks = ('cyclist',)
-        self.save = save
+            self.val_task = 'cyclist'
-        self.print_loss = print_loss
+            self.lambdas = (1,)
-        self.baseline = baseline
+        # Select path out
-        self.lr = lr
+        if args.out:
-        self.sched_step = sched_step
+            self.path_out = args.out  # full path without extension
-        self.sched_gamma = sched_gamma
+            dir_out, _ = os.path.split(self.path_out)
-        n_joints = 17
+        else:
-        input_size = n_joints * 2
+            dir_out = os.path.join('data', 'outputs')
-        self.output_size = 2
+            name = ('monoloco_pp' if self.mode == 'mono' else
-        self.clusters = ['10', '20', '30', '>30']
+                    'monstereo' if self.mode == 'stereo' else
-        self.hidden_size = hidden_size
+                    'casr' if self.mode == 'casr' else 'casr_std')
-        self.n_stage = n_stage
-        self.dir_out = dir_out
-        self.n_samples = n_samples
-        self.r_seed = r_seed
-
-        # Loss functions and output names
             now = datetime.datetime.now()
             now_time = now.strftime("%Y%m%d-%H%M")[2:]
-
+            name_out = name + '-' + now_time + '.pkl'
-        if baseline:
+            self.path_out = os.path.join(dir_out, name_out)
-            name_out = 'baseline-' + now_time
+        assert os.path.exists(dir_out), "Directory to save the model not found"
-            self.criterion = nn.L1Loss().cuda()
+        print(self.path_out)
-            self.output_size = 1
+        # Select the device
-        else:
-            name_out = 'monoloco-' + now_time
-            self.criterion = LaplacianLoss().cuda()
-            self.output_size = 2
-        self.criterion_eval = nn.L1Loss().cuda()
-
-        if self.save:
-            self.path_model = os.path.join(dir_out, name_out + '.pkl')
-            self.logger = set_logger(os.path.join(dir_logs, name_out))
-            self.logger.info("Training arguments: \nepochs: {} \nbatch_size: {} \ndropout: {}"
-                             "\nbaseline: {} \nlearning rate: {} \nscheduler step: {} \nscheduler gamma: {}  "
-                             "\ninput_size: {} \nhidden_size: {} \nn_stages: {} \nr_seed: {}"
-                             "\ninput_file: {}"
-                             .format(epochs, bs, dropout, baseline, lr, sched_step, sched_gamma, input_size,
-                                     hidden_size, n_stage, r_seed, self.joints))
-        else:
-            logging.basicConfig(level=logging.INFO)
-            self.logger = logging.getLogger(__name__)
-
-        # Select the device and load the data
         use_cuda = torch.cuda.is_available()
         self.device = torch.device("cuda" if use_cuda else "cpu")
         print('Device: ', self.device)
-
+        torch.manual_seed(self.r_seed)
-        # Set the seed for random initialization
-        torch.manual_seed(r_seed)
         if use_cuda:
-            torch.cuda.manual_seed(r_seed)
+            torch.cuda.manual_seed(self.r_seed)
+
+        # Remove auxiliary task if monocular
+        if self.mode == 'mono' and self.tasks[-1] == 'aux':
+            self.tasks = self.tasks[:-1]
+            self.lambdas = self.lambdas[:-1]
+
+        losses_tr, losses_val = CompositeLoss(self.tasks)()
+
+        if self.auto_tune_mtl:
+            self.mt_loss = AutoTuneMultiTaskLoss(losses_tr, losses_val, self.lambdas, self.tasks)
+        else:
+            self.mt_loss = MultiTaskLoss(losses_tr, losses_val, self.lambdas, self.tasks)
+        self.mt_loss.to(self.device)
 
         # Dataloader
         self.dataloaders = {phase: DataLoader(KeypointsDataset(self.joints, phase=phase),
-                                              batch_size=bs, shuffle=True) for phase in ['train', 'val']}
+                                              batch_size=args.bs, shuffle=True) for phase in ['train', 'val']}
 
         self.dataset_sizes = {phase: len(KeypointsDataset(self.joints, phase=phase))
-                              for phase in ['train', 'val', 'test']}
+                              for phase in ['train', 'val']}
+        self.dataset_version = KeypointsDataset(self.joints, phase='train').get_version()
+
+        self._set_logger(args)
 
         # Define the model
         self.logger.info('Sizes of the dataset: {}'.format(self.dataset_sizes))
         print(">>> creating model")
-        self.model = LinearModel(input_size=input_size, output_size=self.output_size, linear_size=hidden_size,
+
-                                 p_dropout=dropout, num_stage=self.n_stage)
+        self.model = LocoModel(
+            input_size=self.input_size[self.mode],
+            output_size=self.output_size[self.mode],
+            linear_size=args.hidden_size,
+            p_dropout=args.dropout,
+            num_stage=self.n_stage,
+            device=self.device,
+        )
         self.model.to(self.device)
-        print(">>> total params: {:.2f}M".format(sum(p.numel() for p in self.model.parameters()) / 1000000.0))
+        print(">>> model params: {:.3f}M".format(sum(p.numel() for p in self.model.parameters()) / 1000000.0))
+        print(">>> loss params: {}".format(sum(p.numel() for p in self.mt_loss.parameters())))
 
         # Optimizer and scheduler
-        self.optimizer = torch.optim.Adam(params=self.model.parameters(), lr=lr)
+        all_params = chain(self.model.parameters(), self.mt_loss.parameters())
+        self.optimizer = torch.optim.Adam(params=all_params, lr=args.lr)
+        self.scheduler = lr_scheduler.ReduceLROnPlateau(self.optimizer, 'min')
         self.scheduler = lr_scheduler.StepLR(self.optimizer, step_size=self.sched_step, gamma=self.sched_gamma)
 
     def train(self):
-
-        # Initialize the variable containing model weights
         since = time.time()
         best_model_wts = copy.deepcopy(self.model.state_dict())
         best_acc = 1e6
+        best_training_acc = 1e6
         best_epoch = 0
-        epoch_losses_tr, epoch_losses_val, epoch_norms, epoch_sis = [], [], [], []
+        epoch_losses = defaultdict(lambda: defaultdict(list))
-
         for epoch in range(self.num_epochs):
+            running_loss = defaultdict(lambda: defaultdict(int))
 
             # Each epoch has a training and validation phase
             for phase in ['train', 'val']:
                 if phase == 'train':
-                    self.scheduler.step()
                     self.model.train()  # Set model to training mode
                 else:
                     self.model.eval()  # Set model to evaluate mode
 
-                running_loss_tr = running_loss_eval = norm_tr = bi_tr = 0.0
-
-                # Iterate over data.
                 for inputs, labels, _, _ in self.dataloaders[phase]:
                     inputs = inputs.to(self.device)
                     labels = labels.to(self.device)
-
-                    # zero the parameter gradients
-                    self.optimizer.zero_grad()
-
-                    # forward
-                    # track history if only in train
                     with torch.set_grad_enabled(phase == 'train'):
+                        if phase == 'train':
+                            self.optimizer.zero_grad()
                             outputs = self.model(inputs)
-
+                            loss, _ = self.mt_loss(outputs, labels, phase=phase)
-                        outputs_eval = outputs[:, 0:1] if self.output_size == 2 else outputs
-
-                        loss = self.criterion(outputs, labels)
-                        loss_eval = self.criterion_eval(outputs_eval, labels)  # L1 loss to evaluation
-
-                        # backward + optimize only if in training phase
-                        if phase == 'train':
                             loss.backward()
+                            torch.nn.utils.clip_grad_norm_(self.model.parameters(), 3)
                             self.optimizer.step()
+                            self.scheduler.step()
 
-                    # statistics
-                    running_loss_tr += loss.item() * inputs.size(0)
-                    running_loss_eval += loss_eval.item() * inputs.size(0)
-
-                epoch_loss = running_loss_tr / self.dataset_sizes[phase]
-                epoch_acc = running_loss_eval / self.dataset_sizes[phase]  # Average distance in meters
-                epoch_norm = float(norm_tr / self.dataset_sizes[phase])
-                epoch_si = float(bi_tr / self.dataset_sizes[phase])
-                if phase == 'train':
-                    epoch_losses_tr.append(epoch_loss)
-                    epoch_norms.append(epoch_norm)
-                    epoch_sis.append(epoch_si)
                         else:
-                    epoch_losses_val.append(epoch_acc)
+                            outputs = self.model(inputs)
+                        with torch.no_grad():
+                            loss_eval, loss_values_eval = self.mt_loss(outputs, labels, phase='val')
+                            self.epoch_logs(phase, loss_eval, loss_values_eval, inputs, running_loss)
 
-                if epoch % 5 == 1:
+            self.cout_values(epoch, epoch_losses, running_loss)
-                    sys.stdout.write('\r' + 'Epoch: {:.0f}   Training Loss: {:.3f}   Val Loss {:.3f}'
-                                     .format(epoch, epoch_losses_tr[-1], epoch_losses_val[-1]) + '\t')
 
             # deep copy the model
-                if phase == 'val' and epoch_acc < best_acc:
+
-                    best_acc = epoch_acc
+            if epoch_losses['val'][self.val_task][-1] < best_acc:
+                best_acc = epoch_losses['val'][self.val_task][-1]
+                best_training_acc = epoch_losses['train']['all'][-1]
                 best_epoch = epoch
                 best_model_wts = copy.deepcopy(self.model.state_dict())
 
         time_elapsed = time.time() - since
-        print('\n\n' + '-'*120)
+        print('\n\n' + '-' * 120)
         self.logger.info('Training:\nTraining complete in {:.0f}m {:.0f}s'
                          .format(time_elapsed // 60, time_elapsed % 60))
-        self.logger.info('Best validation Accuracy: {:.3f}'.format(best_acc))
+        self.logger.info('Best training Accuracy: {:.3f}'.format(best_training_acc))
+        self.logger.info('Best validation Accuracy for {}: {:.3f}'.format(self.val_task, best_acc))
         self.logger.info('Saved weights of the model at epoch: {}'.format(best_epoch))
 
-        if self.print_loss:
+        self._print_losses(epoch_losses)
-            epoch_losses_val_scaled = [x - 4 for x in epoch_losses_val]  # to compare with L1 Loss
-            plt.plot(epoch_losses_tr[10:], label='Training Loss')
-            plt.plot(epoch_losses_val_scaled[10:], label='Validation Loss')
-            plt.legend()
-            plt.show()
 
         # load best model weights
         self.model.load_state_dict(best_model_wts)
-
         return best_epoch
 
+    def epoch_logs(self, phase, loss, loss_values, inputs, running_loss):
+
+        running_loss[phase]['all'] += loss.item() * inputs.size(0)
+        for i, task in enumerate(self.tasks):
+            running_loss[phase][task] += loss_values[i].item() * inputs.size(0)
+
     def evaluate(self, load=False, model=None, debug=False):
 
         # To load a model instead of using the trained one
@@ -215,13 +212,12 @@ class Trainer:
         # Average distance on training and test set after unnormalizing
         self.model.eval()
         dic_err = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0)))  # initialized to zero
-        phase = 'val'
+        dic_err['val']['sigmas'] = [0.] * len(self.tasks)
-        batch_size = 5000
+        dataset = KeypointsDataset(self.joints, phase='val')
-        dataset = KeypointsDataset(self.joints, phase=phase)
         size_eval = len(dataset)
         start = 0
         with torch.no_grad():
-            for end in range(batch_size, size_eval+batch_size, batch_size):
+            for end in range(self.VAL_BS, size_eval + self.VAL_BS, self.VAL_BS):
                 end = end if end < size_eval else size_eval
                 inputs, labels, _, _ = dataset[start:end]
                 start = end
@@ -235,18 +231,11 @@ class Trainer:
 
                 # Forward pass
                 outputs = self.model(inputs)
-                if not self.baseline:
+                if not self.is_casr:
-                    outputs = unnormalize_bi(outputs)
+                    self.compute_stats(outputs, labels, dic_err['val'], size_eval, clst='all')
-
-                dic_err[phase]['all'] = self.compute_stats(outputs, labels, dic_err[phase]['all'], size_eval)
-
-            print('-'*120)
-            self.logger.info("Evaluation:\nAverage distance on the {} set: {:.2f}"
-                             .format(phase, dic_err[phase]['all']['mean']))
-
-            self.logger.info("Aleatoric Uncertainty: {:.2f}, inside the interval: {:.1f}%\n"
-                             .format(dic_err[phase]['all']['bi'], dic_err[phase]['all']['conf_bi']*100))
 
+            if not self.is_casr:
+                self.cout_stats(dic_err['val'], size_eval, clst='all')
                 # Evaluate performances on different clusters and save statistics
                 for clst in self.clusters:
                     inputs, labels, size_eval = dataset.get_cluster_annotations(clst)
@@ -254,49 +243,138 @@ class Trainer:
 
                     # Forward pass on each cluster
                     outputs = self.model(inputs)
-                if not self.baseline:
+                    self.compute_stats(outputs, labels, dic_err['val'], size_eval, clst=clst)
-                    outputs = unnormalize_bi(outputs)
+                    self.cout_stats(dic_err['val'], size_eval, clst=clst)
-
-                    dic_err[phase][clst] = self.compute_stats(outputs, labels, dic_err[phase][clst], size_eval)
-
-                self.logger.info("{} error in cluster {} = {:.2f} for {} instances. "
-                                 "Aleatoric of {:.2f} with {:.1f}% inside the interval"
-                                 .format(phase, clst, dic_err[phase][clst]['mean'], size_eval,
-                                         dic_err[phase][clst]['bi'], dic_err[phase][clst]['conf_bi'] * 100))
 
         # Save the model and the results
-        if self.save and not load:
+        if not (self.no_save or load):
             torch.save(self.model.state_dict(), self.path_model)
-            print('-'*120)
+            print('-' * 120)
             self.logger.info("\nmodel saved: {} \n".format(self.path_model))
         else:
             self.logger.info("\nmodel not saved\n")
 
         return dic_err, self.model
 
-    def compute_stats(self, outputs, labels_orig, dic_err, size_eval):
+    def compute_stats(self, outputs, labels, dic_err, size_eval, clst):
         """Compute mean, bi and max of torch tensors"""
 
-        labels = labels_orig.view(-1, )
+        _, loss_values = self.mt_loss(outputs, labels, phase='val')
-        mean_mu = float(self.criterion_eval(outputs[:, 0], labels).item())
+        rel_frac = outputs.size(0) / size_eval
-        max_mu = float(torch.max(torch.abs((outputs[:, 0] - labels))).item())
 
-        if self.baseline:
+        tasks = self.tasks[:-1] if self.tasks[-1] == 'aux' else self.tasks  # Exclude auxiliary
-            return (mean_mu, max_mu), (0, 0, 0)
 
-        mean_bi = torch.mean(outputs[:, 1]).item()
+        for idx, task in enumerate(tasks):
+            dic_err[clst][task] += float(loss_values[idx].item()) * (outputs.size(0) / size_eval)
 
-        low_bound_bi = labels >= (outputs[:, 0] - outputs[:, 1])
+        # Distance
-        up_bound_bi = labels <= (outputs[:, 0] + outputs[:, 1])
+        errs = torch.abs(extract_outputs(outputs)['d'] - extract_labels(labels)['d'])
-        bools_bi = low_bound_bi & up_bound_bi
+        assert rel_frac > 0.99, "Variance of errors not supported with partial evaluation"
-        conf_bi = float(torch.sum(bools_bi)) / float(bools_bi.shape[0])
 
-        dic_err['mean'] += mean_mu * (outputs.size(0) / size_eval)
+        # Uncertainty
-        dic_err['bi'] += mean_bi * (outputs.size(0) / size_eval)
+        bis = extract_outputs(outputs)['bi'].cpu()
-        dic_err['count'] += (outputs.size(0) / size_eval)
+        bi = float(torch.mean(bis).item())
-        dic_err['conf_bi'] += conf_bi * (outputs.size(0) / size_eval)
+        bi_perc = float(torch.sum(errs <= bis)) / errs.shape[0]
+        dic_err[clst]['bi'] += bi * rel_frac
+        dic_err[clst]['bi%'] += bi_perc * rel_frac
+        dic_err[clst]['std'] = errs.std()
 
-        return dic_err
+        # (Don't) Save auxiliary task results
+        if self.mode == 'mono':
+            dic_err[clst]['aux'] = 0
+            dic_err['sigmas'].append(0)
+        elif not self.is_casr:
+            acc_aux = get_accuracy(extract_outputs(outputs)['aux'], extract_labels(labels)['aux'])
+            dic_err[clst]['aux'] += acc_aux * rel_frac
+
+        if self.auto_tune_mtl:
+            assert len(loss_values) == 2 * len(self.tasks)
+            for i, _ in enumerate(self.tasks):
+                dic_err['sigmas'][i] += float(loss_values[len(tasks) + i + 1].item()) * rel_frac
+
+    def cout_stats(self, dic_err, size_eval, clst):
+        if clst == 'all':
+            print('-' * 120)
+            self.logger.info("Evaluation, val set: \nAv. dist D: {:.2f} m with bi {:.2f} ({:.1f}%), \n"
+                             "X: {:.1f} cm,  Y: {:.1f} cm \nOri: {:.1f}  "
+                             "\n H: {:.1f} cm, W: {:.1f} cm, L: {:.1f} cm"
+                             "\nAuxiliary Task: {:.1f} %, "
+                             .format(dic_err[clst]['d'], dic_err[clst]['bi'], dic_err[clst]['bi%'] * 100,
+                                     dic_err[clst]['x'] * 100, dic_err[clst]['y'] * 100,
+                                     dic_err[clst]['ori'], dic_err[clst]['h'] * 100, dic_err[clst]['w'] * 100,
+                                     dic_err[clst]['l'] * 100, dic_err[clst]['aux'] * 100))
+            if self.auto_tune_mtl:
+                self.logger.info("Sigmas: Z: {:.2f}, X: {:.2f}, Y:{:.2f}, H: {:.2f}, W: {:.2f}, L: {:.2f}, ORI: {:.2f}"
+                                 " AUX:{:.2f}\n"
+                                 .format(*dic_err['sigmas']))
+        else:
+            self.logger.info("Val err clust {} --> D:{:.2f}m,  bi:{:.2f} ({:.1f}%), STD:{:.1f}m   X:{:.1f} Y:{:.1f}  "
+                             "Ori:{:.1f}d,   H: {:.0f} W: {:.0f} L:{:.0f}  for {} pp. "
+                             .format(clst, dic_err[clst]['d'], dic_err[clst]['bi'], dic_err[clst]['bi%'] * 100,
+                                     dic_err[clst]['std'], dic_err[clst]['x'] * 100, dic_err[clst]['y'] * 100,
+                                     dic_err[clst]['ori'], dic_err[clst]['h'] * 100, dic_err[clst]['w'] * 100,
+                                     dic_err[clst]['l'] * 100, size_eval))
+
+    def cout_values(self, epoch, epoch_losses, running_loss):
+
+        string = '\r' + '{:.0f} '
+        format_list = [epoch]
+        for phase in running_loss:
+            string = string + phase[0:1].upper() + ':'
+            for el in running_loss['train']:
+                loss = running_loss[phase][el] / self.dataset_sizes[phase]
+                epoch_losses[phase][el].append(loss)
+                if el == 'all':
+                    string = string + ':{:.1f}  '
+                    format_list.append(loss)
+                elif el in ('ori', 'aux'):
+                    string = string + el + ':{:.1f}  '
+                    format_list.append(loss)
+                else:
+                    string = string + el + ':{:.0f}  '
+                    format_list.append(loss * 100)
+
+        if epoch % 10 == 0:
+            print(string.format(*format_list))
+
+    def _print_losses(self, epoch_losses):
+        if not self.print_loss:
+            return
+        os.makedirs(self.dir_figures, exist_ok=True)
+
+        if plt is None:
+            raise Exception('please install matplotlib')
+
+        for idx, phase in enumerate(epoch_losses):
+            for idx_2, el in enumerate(epoch_losses['train']):
+                plt.figure(idx + idx_2)
+                plt.title(phase + '_' + el)
+                plt.xlabel('epochs')
+                plt.plot(epoch_losses[phase][el][10:], label='{} Loss: {}'.format(phase, el))
+                plt.savefig(os.path.join(self.dir_figures, '{}_loss_{}.png'.format(phase, el)))
+                plt.close()
+
+    def _set_logger(self, args):
+        if self.no_save:
+            logging.basicConfig(level=logging.INFO)
+            self.logger = logging.getLogger(__name__)
+        else:
+            self.path_model = self.path_out
+            print(self.path_model)
+            self.logger = set_logger(os.path.splitext(self.path_out)[0])  # remove .pkl
+            self.logger.info(  # pylint: disable=logging-fstring-interpolation
+                f'\nVERSION: {__version__}\n'
+                f'\nINPUT_FILE: {args.joints}'
+                f'\nInput file version: {self.dataset_version}'
+                f'\nTorch version: {torch.__version__}\n'
+                f'\nTraining arguments:'
+                f'\nmode: {self.mode} \nlearning rate: {args.lr} \nbatch_size: {args.bs}'
+                f'\nepochs: {args.epochs} \ndropout: {args.dropout} '
+                f'\nscheduler step: {args.sched_step} \nscheduler gamma: {args.sched_gamma} '
+                f'\ninput_size: {self.input_size[self.mode]} \noutput_size: {self.output_size[self.mode]} '
+                f'\nhidden_size: {args.hidden_size}'
+                f' \nn_stages: {args.n_stage} \n r_seed: {args.r_seed} \nlambdas: {self.lambdas}'
+            )
 
 
 def debug_plots(inputs, labels):
@@ -310,3 +388,11 @@ def debug_plots(inputs, labels):
     plt.figure(2)
     plt.hist(labels, bins='auto')
     plt.show()
+
+
+def get_accuracy(outputs, labels):
+    """From Binary cross entropy outputs to accuracy"""
+
+    mask = outputs >= 0.5
+    accuracy = 1. - torch.mean(torch.abs(mask.float() - labels)).item()
+    return accuracy

monoloco/utils/__init__.py

@@ -1,7 +1,13 @@
 
-from .iou import get_iou_matches, reorder_matches, get_iou_matrix
+from .iou import get_iou_matches, reorder_matches, get_iou_matrix, get_iou_matches_matrix, get_category, \
-from .misc import get_task_error, get_pixel_error, append_cluster, open_annotations
+    open_annotations
-from .kitti import check_conditions, get_category, split_training, parse_ground_truth, get_calibration
+from .misc import get_task_error, get_pixel_error, append_cluster,  make_new_directory,\
-from .camera import xyz_from_distance, get_keypoints, pixel_to_camera, project_3d, open_image
+    normalize_hwl, average
+from .kitti import check_conditions, get_difficulty, split_training, get_calibration, \
+    factory_basename, read_and_rewrite, find_cluster
+from .camera import xyz_from_distance, get_keypoints, pixel_to_camera, project_3d, open_image, correct_angle,\
+    to_spherical, to_cartesian, back_correct_angles, project_to_pixels
 from .logs import set_logger
-from ..utils.nuscenes import select_categories
+from .nuscenes import select_categories
+from .stereo import mask_joint_disparity, average_locations, extract_stereo_matches, \
+    verify_stereo, disparity_to_depth

monoloco/utils/camera.py

@@ -1,4 +1,6 @@
 
+import math
+
 import numpy as np
 import torch
 import torch.nn.functional as F
@@ -30,10 +32,9 @@ def pixel_to_camera(uv_tensor, kk, z_met):
 def project_to_pixels(xyz, kk):
     """Project a single point in space into the image"""
     xx, yy, zz = np.dot(kk, xyz)
-    uu = int(xx / zz)
+    uu = round(xx / zz)
-    vv = int(yy / zz)
+    vv = round(yy / zz)
-
+    return [uu, vv]
-    return uu, vv
 
 
 def project_3d(box_obj, kk):
@@ -180,3 +181,70 @@ def open_image(path_image):
     with open(path_image, 'rb') as f:
         pil_image = Image.open(f).convert('RGB')
         return pil_image
+
+
+def correct_angle(yaw, xyz):
+    """
+    Correct the angle from the egocentric (global/ rotation_y)
+    to allocentric (camera perspective / observation angle)
+    and to be -pi < angle < pi
+    """
+    correction = math.atan2(xyz[0], xyz[2])
+    yaw = yaw - correction
+    if yaw > np.pi:
+        yaw -= 2 * np.pi
+    elif yaw < -np.pi:
+        yaw += 2 * np.pi
+    assert -2 * np.pi <= yaw <= 2 * np.pi
+    return math.sin(yaw), math.cos(yaw), yaw
+
+
+def back_correct_angles(yaws, xyz):
+    corrections = torch.atan2(xyz[:, 0], xyz[:, 2])
+    yaws = yaws + corrections.view(-1, 1)
+    mask_up = yaws > math.pi
+    yaws[mask_up] -= 2 * math.pi
+    mask_down = yaws < -math.pi
+    yaws[mask_down] += 2 * math.pi
+    assert torch.all(yaws < math.pi) & torch.all(yaws > - math.pi)
+    return yaws
+
+
+def to_spherical(xyz):
+    """convert from cartesian to spherical"""
+    xyz = np.array(xyz)
+    r = np.linalg.norm(xyz)
+    theta = math.atan2(xyz[2], xyz[0])
+
+    assert 0 <= theta < math.pi   # 0 when positive x and no z.
+    psi = math.acos(xyz[1] / r)
+    assert 0 <= psi <= math.pi
+    return [r, theta, psi]
+
+
+def to_cartesian(rtp, mode=None):
+    """convert from spherical to cartesian"""
+
+    if isinstance(rtp, torch.Tensor):
+        if mode in ('x', 'y'):
+            r = rtp[:, 2]
+            t = rtp[:, 0]
+            p = rtp[:, 1]
+        if mode == 'x':
+            x = r * torch.sin(p) * torch.cos(t)
+            return x.view(-1, 1)
+
+        if mode == 'y':
+            y = r * torch.cos(p)
+            return y.view(-1, 1)
+
+        xyz = rtp.clone()
+        xyz[:, 0] = rtp[:, 0] * torch.sin(rtp[:, 2]) * torch.cos(rtp[:, 1])
+        xyz[:, 1] = rtp[:, 0] * torch.cos(rtp[:, 2])
+        xyz[:, 2] = rtp[:, 0] * torch.sin(rtp[:, 2]) * torch.sin(rtp[:, 1])
+        return xyz
+
+    x = rtp[0] * math.sin(rtp[2]) * math.cos(rtp[1])
+    y = rtp[0] * math.cos(rtp[2])
+    z = rtp[0] * math.sin(rtp[2]) * math.sin(rtp[1])
+    return[x, y, z]

monoloco/utils/iou.py

@@ -1,10 +1,17 @@
 
+import json
+
 import numpy as np
 
 
 def calculate_iou(box1, box2):
 
     # Calculate the (x1, y1, x2, y2) coordinates of the intersection of box1 and box2. Calculate its Area.
+    # box1 = [-3, 8.5, 3, 11.5]
+    # box2 = [-3, 9.5, 3, 12.5]
+    # box1 = [1086.84, 156.24, 1181.62, 319.12]
+    # box2 = [1078.333357, 159.086347, 1193.771014, 322.239107]
+
     xi1 = max(box1[0], box2[0])
     yi1 = max(box1[1], box2[1])
     xi2 = min(box1[2], box2[2])
@@ -34,7 +41,30 @@ def get_iou_matrix(boxes, boxes_gt):
     return iou_matrix
 
 
-def get_iou_matches(boxes, boxes_gt, thresh):
+def get_iou_matches(boxes, boxes_gt, iou_min=0.3):
+    """From 2 sets of boxes and a minimum threshold, compute the matching indices for IoU matches"""
+
+    matches = []
+    used = []
+    if not boxes or not boxes_gt:
+        return []
+    confs = [box[4] for box in boxes]
+
+    indices = list(np.argsort(confs))
+    for idx in indices[::-1]:
+        box = boxes[idx]
+        ious = []
+        for box_gt in boxes_gt:
+            iou = calculate_iou(box, box_gt)
+            ious.append(iou)
+        idx_gt_max = int(np.argmax(ious))
+        if (ious[idx_gt_max] >= iou_min) and (idx_gt_max not in used):
+            matches.append((int(idx), idx_gt_max))
+            used.append(idx_gt_max)
+    return matches
+
+
+def get_iou_matches_matrix(boxes, boxes_gt, thresh):
     """From 2 sets of boxes and a minimum threshold, compute the matching indices for IoU matchings"""
 
     iou_matrix = get_iou_matrix(boxes, boxes_gt)
@@ -65,6 +95,51 @@ def reorder_matches(matches, boxes, mode='left_rigth'):
 
     # Order the boxes based on the left-right position in the image and
     ordered_boxes = np.argsort([box[0] for box in boxes])  # indices of boxes ordered from left to right
-    matches_left = [idx for (idx, _) in matches]
+    matches_left = [int(idx) for (idx, _) in matches]
 
     return [matches[matches_left.index(idx_boxes)] for idx_boxes in ordered_boxes if idx_boxes in matches_left]
+
+
+def get_category(keypoints, path_byc):
+    """Find the category for each of the keypoints"""
+
+    dic_byc = open_annotations(path_byc)
+    boxes_byc = dic_byc['boxes'] if dic_byc else []
+    boxes_ped = make_lower_boxes(keypoints)
+
+    matches = get_matches_bikes(boxes_ped, boxes_byc)
+    list_byc = [match[0] for match in matches]
+    categories = [1.0 if idx in list_byc else 0.0 for idx, _ in enumerate(boxes_ped)]
+    return categories
+
+
+def get_matches_bikes(boxes_ped, boxes_byc):
+    matches = get_iou_matches_matrix(boxes_ped, boxes_byc, thresh=0.15)
+    matches_b = []
+    for idx, idx_byc in matches:
+        box_ped = boxes_ped[idx]
+        box_byc = boxes_byc[idx_byc]
+        width_ped = box_ped[2] - box_ped[0]
+        width_byc = box_byc[2] - box_byc[0]
+        center_ped = (box_ped[2] + box_ped[0]) / 2
+        center_byc = (box_byc[2] + box_byc[0]) / 2
+        if abs(center_ped - center_byc) < min(width_ped, width_byc) / 4:
+            matches_b.append((idx, idx_byc))
+    return matches_b
+
+
+def make_lower_boxes(keypoints):
+    lower_boxes = []
+    keypoints = np.array(keypoints)
+    for kps in keypoints:
+        lower_boxes.append([min(kps[0, 9:]), min(kps[1, 9:]), max(kps[0, 9:]), max(kps[1, 9:])])
+    return lower_boxes
+
+
+def open_annotations(path_ann):
+    try:
+        with open(path_ann, 'r') as f:
+            annotations = json.load(f)
+    except FileNotFoundError:
+        annotations = []
+    return annotations

monoloco/utils/kitti.py

@@ -1,5 +1,6 @@
 
-import math
+import os
+import glob
 
 import numpy as np
 
@@ -76,32 +77,21 @@ def check_conditions(line, category, method, thresh=0.3):
     check = False
     assert category in ['pedestrian', 'cyclist', 'all']
 
-    if method == 'gt':
     if category == 'all':
-            categories_gt = ['Pedestrian', 'Person_sitting', 'Cyclist']
+        category = ['pedestrian', 'person_sitting', 'cyclist']
-        else:
-            categories_gt = [category.upper()[0] + category[1:]]  # Upper case names
-        if line.split()[0] in categories_gt:
-            check = True
 
-    elif method in ('m3d', '3dop'):
+    if method == 'gt':
-        conf = float(line[15])
+        if line.split()[0].lower() in category:
-        if line[0] == category and conf >= thresh:
-            check = True
-
-    elif method == 'monodepth':
             check = True
 
     else:
-        zz = float(line[13])
         conf = float(line[15])
-        if conf >= thresh and 0.5 < zz < 70:
+        if line[0].lower() in category and conf >= thresh:
             check = True
-
     return check
 
 
-def get_category(box, trunc, occ):
+def get_difficulty(box, trunc, occ):
 
     hh = box[3] - box[1]
     if hh >= 40 and trunc <= 0.15 and occ <= 0:
@@ -128,31 +118,57 @@ def split_training(names_gt, path_train, path_val):
         for line in f_val:
             set_val.add(line[:-1] + '.txt')
 
-    set_train = tuple(set_gt.intersection(set_train))
+    set_train = set_gt.intersection(set_train)
+    set_train.remove('000518.txt')
+    set_train.remove('005692.txt')
+    set_train.remove('003009.txt')
+    set_train = tuple(set_train)
     set_val = tuple(set_gt.intersection(set_val))
     assert set_train and set_val, "No validation or training annotations"
     return set_train, set_val
 
 
-def parse_ground_truth(path_gt, category):
+def factory_basename(dir_ann, dir_gt):
-    """Parse KITTI ground truth files"""
+    """ Return all the basenames in the annotations folder corresponding to validation images"""
-    boxes_gt = []
-    dds_gt = []
-    zzs_gt = []
-    truncs_gt = []  # Float from 0 to 1
-    occs_gt = []  # Either 0,1,2,3 fully visible, partly occluded, largely occluded, unknown
-    boxes_3d = []
 
-    with open(path_gt, "r") as f_gt:
+    # Extract ground truth validation images
+    names_gt = tuple(os.listdir(dir_gt))
+    path_train = os.path.join('splits', 'kitti_train.txt')
+    path_val = os.path.join('splits', 'kitti_val.txt')
+    _, set_val_gt = split_training(names_gt, path_train, path_val)
+    set_val_gt = {os.path.basename(x).split('.')[0] for x in set_val_gt}
+
+    # Extract pifpaf files corresponding to validation images
+    list_ann = glob.glob(os.path.join(dir_ann, '*.json'))
+    set_basename = {os.path.basename(x).split('.')[0] for x in list_ann}
+    set_val = set_basename.intersection(set_val_gt)
+    assert set_val, " Missing json annotations file to create txt files for KITTI datasets"
+    return set_val
+
+
+def read_and_rewrite(path_orig, path_new):
+    """Read and write same txt file. If file not found, create open file"""
+    try:
+        with open(path_orig, "r") as f_gt:
+            with open(path_new, "w+") as ff:
                 for line_gt in f_gt:
-            if check_conditions(line_gt, category, method='gt'):
+                    # if check_conditions(line_gt, category='all', method='gt'):
-                truncs_gt.append(float(line_gt.split()[1]))
+                    line = line_gt.split()
-                occs_gt.append(int(line_gt.split()[2]))
+                    hwl = [float(x) for x in line[8:11]]
-                boxes_gt.append([float(x) for x in line_gt.split()[4:8]])
+                    hwl = " ".join([str(i)[0:4] for i in hwl])
-                loc_gt = [float(x) for x in line_gt.split()[11:14]]
+                    temp_1 = " ".join([str(i) for i in line[0: 8]])
-                wlh = [float(x) for x in line_gt.split()[8:11]]
+                    temp_2 = " ".join([str(i) for i in line[11:]])
-                boxes_3d.append(loc_gt + wlh)
+                    line_new = temp_1 + ' ' + hwl + ' ' + temp_2 + '\n'
-                zzs_gt.append(loc_gt[2])
+                    ff.write("%s" % line_new)
-                dds_gt.append(math.sqrt(loc_gt[0] ** 2 + loc_gt[1] ** 2 + loc_gt[2] ** 2))
+    except FileNotFoundError:
+        with open(path_new, "a+"):
+            pass
 
-    return boxes_gt, boxes_3d, dds_gt, zzs_gt, truncs_gt, occs_gt
+
+def find_cluster(dd, clusters):
+    """Find the correct cluster. Above the last cluster goes into "excluded (together with the ones from kitti cat"""
+
+    for idx, clst in enumerate(clusters[:-1]):
+        if int(clst) < dd <= int(clusters[idx+1]):
+            return clst
+    return 'excluded'

monoloco/utils/misc.py

@@ -1,28 +1,32 @@
-import json
+import shutil
+import os
+
+import numpy as np
 
 
-def append_cluster(dic_jo, phase, xx, dd, kps):
+def append_cluster(dic_jo, phase, xx, ys, kps):
     """Append the annotation based on its distance"""
 
-    if dd <= 10:
+    if ys[3] <= 10:
         dic_jo[phase]['clst']['10']['kps'].append(kps)
         dic_jo[phase]['clst']['10']['X'].append(xx)
-        dic_jo[phase]['clst']['10']['Y'].append([dd])
+        dic_jo[phase]['clst']['10']['Y'].append(ys)
-
+    elif ys[3] <= 20:
-    elif dd <= 20:
         dic_jo[phase]['clst']['20']['kps'].append(kps)
         dic_jo[phase]['clst']['20']['X'].append(xx)
-        dic_jo[phase]['clst']['20']['Y'].append([dd])
+        dic_jo[phase]['clst']['20']['Y'].append(ys)
-
+    elif ys[3] <= 30:
-    elif dd <= 30:
         dic_jo[phase]['clst']['30']['kps'].append(kps)
         dic_jo[phase]['clst']['30']['X'].append(xx)
-        dic_jo[phase]['clst']['30']['Y'].append([dd])
+        dic_jo[phase]['clst']['30']['Y'].append(ys)
-
+    elif ys[3] <= 40:
+        dic_jo[phase]['clst']['40']['kps'].append(kps)
+        dic_jo[phase]['clst']['40']['X'].append(xx)
+        dic_jo[phase]['clst']['40']['Y'].append(ys)
     else:
-        dic_jo[phase]['clst']['>30']['kps'].append(kps)
+        dic_jo[phase]['clst']['>40']['kps'].append(kps)
-        dic_jo[phase]['clst']['>30']['X'].append(xx)
+        dic_jo[phase]['clst']['>40']['X'].append(xx)
-        dic_jo[phase]['clst']['>30']['Y'].append([dd])
+        dic_jo[phase]['clst']['>40']['Y'].append(ys)
 
 
 def get_task_error(dd):
@@ -39,10 +43,27 @@ def get_pixel_error(zz_gt):
     return error
 
 
-def open_annotations(path_ann):
+def make_new_directory(dir_out):
-    try:
+    """Remove the output directory if already exists (avoid residual txt files)"""
-        with open(path_ann, 'r') as f:
+    if os.path.exists(dir_out):
-            annotations = json.load(f)
+        shutil.rmtree(dir_out)
-    except FileNotFoundError:
+    os.makedirs(dir_out)
-        annotations = []
+    print("Created empty output directory {} ".format(dir_out))
-    return annotations
+
+
+def normalize_hwl(lab):
+
+    AV_H = 1.72
+    AV_W = 0.75
+    AV_L = 0.68
+    HLW_STD = 0.1
+
+    hwl = lab[4:7]
+    hwl_new = list((np.array(hwl) - np.array([AV_H, AV_W, AV_L])) / HLW_STD)
+    lab_new = lab[0:4] + hwl_new + lab[7:]
+    return lab_new
+
+
+def average(my_list):
+    """calculate mean of a list"""
+    return sum(my_list) / len(my_list)

monoloco/utils/stereo.py

@@ -0,0 +1,197 @@
+
+import warnings
+
+import numpy as np
+
+
+BF = 0.54 * 721
+z_min = 4
+z_max = 60
+D_MIN = BF / z_max
+D_MAX = BF / z_min
+
+
+def extract_stereo_matches(keypoint, keypoints_r, zz, phase='train', seed=0, method=None):
+    """
+    Return:
+        1) a list of tuples that indicates, for a reference pose in the left image:
+    - the index of the right pose
+    - weather the right pose corresponds to the same person as the left pose (stereo match) or not
+    For example: [(0,0), (1,0), (2,1)] means there are three right poses in the image
+    and the third one is the same person as the reference pose
+    2) a flag indicating whether a match has been found
+    3) number of ambiguous instances, for which is not possible to define whether there is a correspondence
+    """
+
+    stereo_matches = []
+    cnt_ambiguous = 0
+    if method == 'mask':
+        conf_min = 0.1
+    else:
+        conf_min = 0.2
+    avgs_x_l, avgs_x_r, disparities_x, disparities_y = average_locations(keypoint, keypoints_r, conf_min=conf_min)
+    avg_disparities = [abs(float(l) - BF / zz - float(r)) for l, r in zip(avgs_x_l, avgs_x_r)]
+    idx_matches = np.argsort(avg_disparities)
+    # error_max_stereo = 1 * 0.0028 * zz**2 + 0.2  # 2m at 20 meters of depth + 20 cm of offset
+    error_max_stereo = 0.2 * zz + 0.2  # 2m at 20 meters of depth + 20 cm of offset
+    error_min_mono = 0.25 * zz + 0.2
+    error_max_mono = 1 * zz + 0.5
+    used = []
+    # Add positive and negative samples
+    for idx, idx_match in enumerate(idx_matches):
+        match = avg_disparities[idx_match]
+        zz_stereo, flag = disparity_to_depth(match + BF / zz)
+
+        # Conditions to accept stereo match
+        conditions = (idx == 0
+                      and match < depth_to_pixel_error(zz, depth_error=error_max_stereo)
+                      and flag
+                      and verify_stereo(zz_stereo, zz, disparities_x[idx_match], disparities_y[idx_match]))
+
+        # Positive matches
+        if conditions:
+            stereo_matches.append((idx_match, 1))
+        # Ambiguous
+        elif match < depth_to_pixel_error(zz, depth_error=error_min_mono):
+            cnt_ambiguous += 1
+
+        # Disparity-range negative
+        # elif D_MIN < match + BF / zz < D_MAX:
+        #     stereo_matches.append((idx_match, 0))
+
+        elif phase == 'val' \
+                and match < depth_to_pixel_error(zz, depth_error=error_max_mono) \
+                and not stereo_matches\
+                and zz < 40:
+            stereo_matches.append((idx_match, 0))
+
+        # # Hard-negative for training
+        elif phase == 'train' \
+                and match < depth_to_pixel_error(zz, depth_error=error_max_mono) \
+                and len(stereo_matches) < 3:
+            stereo_matches.append((idx_match, 0))
+
+        # # Easy-negative
+        elif phase == 'train' \
+                and len(stereo_matches) < 3:
+            np.random.seed(seed + idx)
+            num = np.random.randint(idx, len(idx_matches))
+            if idx_matches[num] not in used:
+                stereo_matches.append((idx_matches[num], 0))
+
+        else:
+            break
+        used.append(idx_match)
+
+    return stereo_matches, cnt_ambiguous
+
+
+def depth_to_pixel_error(zz, depth_error=1):
+    """
+    Calculate the pixel error at a certain depth due to depth error according to:
+    e_d = b * f * e_z / (z**2)
+    """
+    e_d = BF * depth_error / (zz**2)
+    return e_d
+
+
+def mask_joint_disparity(keypoints, keypoints_r):
+    """filter joints based on confidence and interquartile range of the distribution"""
+    # TODO Merge with average location
+    CONF_MIN = 0.3
+    with warnings.catch_warnings() and np.errstate(invalid='ignore'):
+        disparity_x_mask = np.empty((keypoints.shape[0], keypoints_r.shape[0], 17))
+        disparity_y_mask = np.empty((keypoints.shape[0], keypoints_r.shape[0], 17))
+        avg_disparity = np.empty((keypoints.shape[0], keypoints_r.shape[0]))
+
+        for idx, kps in enumerate(keypoints):
+            disparity_x = kps[0, :] - keypoints_r[:, 0, :]
+            disparity_y = kps[1, :] - keypoints_r[:, 1, :]
+
+            # Mask for low confidence
+            mask_conf_left = kps[2, :] > CONF_MIN
+            mask_conf_right = keypoints_r[:, 2, :] > CONF_MIN
+            mask_conf = mask_conf_left & mask_conf_right
+            disparity_x_conf = np.where(mask_conf, disparity_x, np.nan)
+            disparity_y_conf = np.where(mask_conf, disparity_y, np.nan)
+
+            # Mask outliers using iqr
+            mask_outlier = interquartile_mask(disparity_x_conf)
+            x_mask_row = np.where(mask_outlier, disparity_x_conf, np.nan)
+            y_mask_row = np.where(mask_outlier, disparity_y_conf, np.nan)
+            avg_row = np.nanmedian(x_mask_row, axis=1)  # ignore the nan
+
+            # Append
+            disparity_x_mask[idx] = x_mask_row
+            disparity_y_mask[idx] = y_mask_row
+            avg_disparity[idx] = avg_row
+
+        return avg_disparity, disparity_x_mask, disparity_y_mask
+
+
+def average_locations(keypoint, keypoints_r, conf_min=0.2):
+    """
+    Extract absolute average location of keypoints
+    INPUT: arrays of (1, 3, 17) & (m,3,17)
+    OUTPUT: 2 arrays of (m).
+    The left keypoint will have different absolute positions based on the right keypoints they are paired with
+    """
+    keypoint, keypoints_r = np.array(keypoint), np.array(keypoints_r)
+    assert keypoints_r.shape[0] > 0, "No right keypoints"
+    with warnings.catch_warnings() and np.errstate(invalid='ignore'):
+
+        # Mask by confidence
+        mask_l_conf = keypoint[0, 2, :] > conf_min
+        mask_r_conf = keypoints_r[:, 2, :] > conf_min
+        abs_x_l = np.where(mask_l_conf, keypoint[0, 0:1, :], np.nan)
+        abs_x_r = np.where(mask_r_conf, keypoints_r[:, 0, :], np.nan)
+
+        # Mask by iqr
+        mask_l_iqr = interquartile_mask(abs_x_l)
+        mask_r_iqr = interquartile_mask(abs_x_r)
+
+        # Combine masks
+        mask = mask_l_iqr & mask_r_iqr
+
+        # Compute absolute locations and relative disparities
+        x_l = np.where(mask, abs_x_l, np.nan)
+        x_r = np.where(mask, abs_x_r, np.nan)
+        x_disp = x_l - x_r
+        y_disp = np.where(mask, keypoint[0, 1, :] - keypoints_r[:, 1, :], np.nan)
+        avgs_x_l = np.nanmedian(x_l, axis=1)
+        avgs_x_r = np.nanmedian(x_r, axis=1)
+
+        return avgs_x_l, avgs_x_r, x_disp, y_disp
+
+
+def interquartile_mask(distribution):
+    quartile_1, quartile_3 = np.nanpercentile(distribution, [25, 75], axis=1)
+    iqr = quartile_3 - quartile_1
+    lower_bound = quartile_1 - (iqr * 1.5)
+    upper_bound = quartile_3 + (iqr * 1.5)
+    return (distribution < upper_bound.reshape(-1, 1)) & (distribution > lower_bound.reshape(-1, 1))
+
+
+def disparity_to_depth(avg_disparity):
+
+    try:
+        zz_stereo = 0.54 * 721. / float(avg_disparity)
+        flag = True
+    except (ZeroDivisionError, ValueError):  # All nan-slices or zero division
+        zz_stereo = np.nan
+        flag = False
+    return zz_stereo, flag
+
+
+def verify_stereo(zz_stereo, zz_mono, disparity_x, disparity_y):
+    """Verify disparities based on coefficient of variation, maximum y difference and z difference wrt monoloco"""
+
+    # COV_MIN = 0.1
+    y_max_difference = (80 / zz_mono)
+    z_max_difference = 1 * zz_mono
+
+    cov = float(np.nanstd(disparity_x) / np.abs(np.nanmean(disparity_x)))  # pylint: disable=unused-variable
+    avg_disparity_y = np.nanmedian(disparity_y)
+
+    return abs(zz_stereo - zz_mono) < z_max_difference and avg_disparity_y < y_max_difference and 1 < zz_stereo < 80
+# cov < COV_MIN and \

monoloco/visuals/__init__.py

@@ -1,3 +1,3 @@
 
 from .printer import Printer
-from .figures import show_results, show_spread, show_task_error
+from .figures import show_results, show_spread, show_task_error, show_box_plot

monoloco/visuals/figures.py

@@ -7,114 +7,125 @@ import os
 import numpy as np
 import matplotlib.pyplot as plt
 from matplotlib.patches import Ellipse
+try:
+    import pandas as pd
+    DATAFRAME = pd.DataFrame
+except ImportError:
+    DATAFRAME = None
 
 from ..utils import get_task_error, get_pixel_error
 
 
-def show_results(dic_stats, show=False, save=False, stereo=False):
+FONTSIZE = 15
+FIGSIZE = (9.6, 7.2)
+DPI = 200
+GRID_WIDTH = 0.5
+
+
+def show_results(dic_stats, clusters, net, dir_fig, show=False, save=False):
     """
     Visualize error as function of the distance and compare it with target errors based on human height analyses
     """
 
-    dir_out = 'docs'
     phase = 'test'
-    x_min = 0
+    x_min = 3
-    x_max = 38
+    # x_max = 42
+    x_max = 31
     y_min = 0
-    y_max = 4.7
+    # y_max = 2.2
-    xx = np.linspace(0, 60, 100)
+    y_max = 3.5 if net == 'monstereo' else 2.7
-    excl_clusters = ['all', '50', '>50', 'easy', 'moderate', 'hard']
+    xx = np.linspace(x_min, x_max, 100)
-    clusters = tuple([clst for clst in dic_stats[phase]['monoloco'] if clst not in excl_clusters])
+    excl_clusters = ['all', 'easy', 'moderate', 'hard', '49']
-    yy_gender = get_task_error(xx)
+    clusters = [clst for clst in clusters if clst not in excl_clusters]
-
+    styles = printing_styles(net)
-    styles = printing_styles(stereo)
+    for idx_style in styles:
-    for idx_style, (key, style) in enumerate(styles.items()):
+        plt.figure(idx_style, figsize=FIGSIZE)
-        plt.figure(idx_style)
+        plt.grid(linewidth=GRID_WIDTH)
-        plt.grid(linewidth=0.2)
         plt.xlim(x_min, x_max)
         plt.ylim(y_min, y_max)
-        plt.xlabel("Ground-truth distance [m]")
+        plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE)
-        plt.ylabel("Average localization error [m]")
+        plt.ylabel("Average localization error (ALE) [m]", fontsize=FONTSIZE)
-        for idx, method in enumerate(style['methods']):
+        for idx, method in enumerate(styles['methods']):
-            errs = [dic_stats[phase][method][clst]['mean'] for clst in clusters]
+            errs = [dic_stats[phase][method][clst]['mean'] for clst in clusters[:-1]]  # last cluster only a bound
+            cnts = [dic_stats[phase][method][clst]['cnt'] for clst in clusters[:-1]]  # last cluster only a bound
             assert errs, "method %s empty" % method
             xxs = get_distances(clusters)
 
-            plt.plot(xxs, errs, marker=style['mks'][idx], markersize=style['mksizes'][idx], linewidth=style['lws'][idx],
+            plt.plot(xxs, errs, marker=styles['mks'][idx], markersize=styles['mksizes'][idx],
-                     label=style['labels'][idx], linestyle=style['lstyles'][idx], color=style['colors'][idx])
+                     linewidth=styles['lws'][idx],
-        plt.plot(xx, yy_gender, '--', label="Task error", color='lightgreen', linewidth=2.5)
+                     label=styles['labels'][idx], linestyle=styles['lstyles'][idx], color=styles['colors'][idx])
-        if key == 'stereo':
+            if method in ('monstereo', 'monoloco_pp', 'pseudo-lidar'):
-            yy_stereo = get_pixel_error(xx)
+                for i, x in enumerate(xxs):
-            plt.plot(xx, yy_stereo, linewidth=1.7, color='k', label='Pixel error')
+                    plt.text(x, errs[i] - 0.1, str(cnts[i]), fontsize=FONTSIZE)
+    if net == 'monoloco_pp':
+        plt.plot(xx, get_task_error(xx), '--', label="Task error", color='lightgreen', linewidth=2.5)
+    # if stereo:
+    #     yy_stereo = get_pixel_error(xx)
+    #     plt.plot(xx, yy_stereo, linewidth=1.4, color='k', label='Pixel error')
 
-        plt.legend(loc='upper left')
+    plt.legend(loc='upper left', prop={'size': FONTSIZE})
+    plt.xticks(fontsize=FONTSIZE)
+    plt.yticks(fontsize=FONTSIZE)
     if save:
-            path_fig = os.path.join(dir_out, 'results_' + key + '.png')
+        plt.tight_layout()
-            plt.savefig(path_fig)
+        path_fig = os.path.join(dir_fig, 'results_' + net + '.png')
-            print("Figure of results " + key + " saved in {}".format(path_fig))
+        plt.savefig(path_fig, dpi=DPI)
+        print("Figure of results " + net + " saved in {}".format(path_fig))
     if show:
         plt.show()
-        plt.close()
+    plt.close('all')
 
 
-def show_spread(dic_stats, show=False, save=False):
+def show_spread(dic_stats, clusters, net, dir_fig, show=False, save=False):
     """Predicted confidence intervals and task error as a function of ground-truth distance"""
 
+    assert net in ('monoloco_pp', 'monstereo'), "network not recognized"
     phase = 'test'
-    dir_out = 'docs'
+    excl_clusters = ['all', 'easy', 'moderate', 'hard', '49']
-    excl_clusters = ['all', '50', '>50', 'easy', 'moderate', 'hard']
+    clusters = [clst for clst in clusters if clst not in excl_clusters]
-    clusters = tuple([clst for clst in dic_stats[phase]['our'] if clst not in excl_clusters])
+    x_min = 3
+    x_max = 31
+    y_min = 0
 
-    plt.figure(2)
+    plt.figure(2, figsize=FIGSIZE)
-    fig, ax = plt.subplots(2, sharex=True)
-    plt.xlabel("Distance [m]")
-    plt.ylabel("Aleatoric uncertainty [m]")
-    ar = 0.5  # Change aspect ratio of ellipses
-    scale = 1.5  # Factor to scale ellipses
-    rec_c = 0  # Center of the rectangle
-    plots_line = True
-
-    bbs = np.array([dic_stats[phase]['our'][key]['std_ale'] for key in clusters])
     xxs = get_distances(clusters)
-    yys = get_task_error(np.array(xxs))
+    bbs = np.array([dic_stats[phase][net][key]['std_ale'] for key in clusters[:-1]])
-    ax[1].plot(xxs, bbs, marker='s', color='b', label="Spread b")
+    xx = np.linspace(x_min, x_max, 100)
-    ax[1].plot(xxs, yys, '--', color='lightgreen', label="Task error", linewidth=2.5)
+    if net == 'monoloco_pp':
-    yys_up = [rec_c + ar / 2 * scale * yy for yy in yys]
+        y_max = 2.7
-    bbs_up = [rec_c + ar / 2 * scale * bb for bb in bbs]
+        color = 'deepskyblue'
-    yys_down = [rec_c - ar / 2 * scale * yy for yy in yys]
+        epis = np.array([dic_stats[phase][net][key]['std_epi'] for key in clusters[:-1]])
-    bbs_down = [rec_c - ar / 2 * scale * bb for bb in bbs]
+        plt.plot(xxs, epis, marker='o', color='coral', linewidth=4, markersize=8, label="Combined uncertainty (\u03C3)")
+    else:
+        y_max = 3.5
+        color = 'b'
+        plt.plot(xx, get_pixel_error(xx), linewidth=2.5, color='k', label='Pixel error')
+    plt.plot(xxs, bbs, marker='s', color=color, label="Aleatoric uncertainty (b)", linewidth=4, markersize=8)
+    plt.plot(xx, get_task_error(xx), '--', label="Task error (monocular bound)", color='lightgreen', linewidth=4)
 
-    if plots_line:
+    plt.xlabel("Ground-truth distance [m]", fontsize=FONTSIZE)
-        ax[0].plot(xxs, yys_up, '--', color='lightgreen', markersize=5, linewidth=1.4)
+    plt.ylabel("Uncertainty [m]", fontsize=FONTSIZE)
-        ax[0].plot(xxs, yys_down, '--', color='lightgreen', markersize=5, linewidth=1.4)
+    plt.xlim(x_min, x_max)
-        ax[0].plot(xxs, bbs_up, marker='s', color='b', markersize=5, linewidth=0.7)
+    plt.ylim(y_min, y_max)
-        ax[0].plot(xxs, bbs_down, marker='s', color='b', markersize=5, linewidth=0.7)
+    plt.grid(linewidth=GRID_WIDTH)
+    plt.legend(prop={'size': FONTSIZE})
+    plt.xticks(fontsize=FONTSIZE)
+    plt.yticks(fontsize=FONTSIZE)
 
-    for idx, xx in enumerate(xxs):
-        te = Ellipse((xx, rec_c), width=yys[idx] * ar * scale, height=scale, angle=90, color='lightgreen', fill=True)
-        bi = Ellipse((xx, rec_c), width=bbs[idx] * ar * scale, height=scale, angle=90, color='b', linewidth=1.8,
-                     fill=False)
-
-        ax[0].add_patch(te)
-        ax[0].add_patch(bi)
-
-    fig.subplots_adjust(hspace=0.1)
-    plt.setp([aa.get_yticklabels() for aa in fig.axes[:-1]], visible=False)
-    plt.legend()
     if save:
-        path_fig = os.path.join(dir_out, 'spread.png')
+        plt.tight_layout()
-        plt.savefig(path_fig)
+        path_fig = os.path.join(dir_fig, 'spread_' + net + '.png')
+        plt.savefig(path_fig, dpi=DPI)
         print("Figure of confidence intervals saved in {}".format(path_fig))
     if show:
         plt.show()
-    plt.close()
+    plt.close('all')
 
 
-def show_task_error(show, save):
+def show_task_error(dir_fig, show, save):
     """Task error figure"""
-    plt.figure(3)
+    plt.figure(3, figsize=FIGSIZE)
-    dir_out = 'docs'
+    xx = np.linspace(0.1, 40, 100)
-    xx = np.linspace(0.1, 50, 100)
     mu_men = 178
     mu_women = 165
     mu_child_m = 164
@@ -128,31 +139,33 @@ def show_task_error(show, save):
     yy_young_female = target_error(xx, mm_young_female)
     yy_gender = target_error(xx, mm_gmm)
     yy_stereo = get_pixel_error(xx)
-    plt.grid(linewidth=0.3)
+    plt.grid(linewidth=GRID_WIDTH)
     plt.plot(xx, yy_young_male, linestyle='dotted', linewidth=2.1, color='b', label='Adult/young male')
     plt.plot(xx, yy_young_female, linestyle='dotted', linewidth=2.1, color='darkorange', label='Adult/young female')
     plt.plot(xx, yy_gender, '--', color='lightgreen', linewidth=2.8, label='Generic adult (task error)')
     plt.plot(xx, yy_female, '-.', linewidth=1.7, color='darkorange', label='Adult female')
     plt.plot(xx, yy_male, '-.', linewidth=1.7, color='b', label='Adult male')
-    plt.plot(xx, yy_stereo, linewidth=1.7, color='k', label='Pixel error')
+    plt.plot(xx, yy_stereo, linewidth=2.5, color='k', label='Pixel error')
     plt.xlim(np.min(xx), np.max(xx))
+    plt.ylim(0, 5)
     plt.xlabel("Ground-truth distance from the camera $d_{gt}$ [m]")
     plt.ylabel("Localization error $\hat{e}$  due to human height variation [m]")  # pylint: disable=W1401
     plt.legend(loc=(0.01, 0.55))  # Location from 0 to 1 from lower left
+    plt.xticks(fontsize=FONTSIZE)
+    plt.yticks(fontsize=FONTSIZE)
     if save:
-        path_fig = os.path.join(dir_out, 'task_error.png')
+        path_fig = os.path.join(dir_fig, 'task_error.png')
-        plt.savefig(path_fig)
+        plt.savefig(path_fig, dpi=DPI)
         print("Figure of task error saved in {}".format(path_fig))
     if show:
         plt.show()
-    plt.close()
+    plt.close('all')
 
 
-def show_method(save):
+def show_method(save, dir_out='data/figures'):
     """ method figure"""
-    dir_out = 'docs'
     std_1 = 0.75
-    fig = plt.figure(1)
+    fig = plt.figure(4, figsize=FIGSIZE)
     ax = fig.add_subplot(1, 1, 1)
     ell_3 = Ellipse((0, 2), width=std_1 * 2, height=0.3, angle=-90, color='b', fill=False, linewidth=2.5)
     ell_4 = Ellipse((0, 2), width=std_1 * 3, height=0.3, angle=-90, color='r', fill=False,
@@ -164,14 +177,47 @@ def show_method(save):
     plt.plot([0, -3], [0, 4], 'k--')
     plt.xlim(-3, 3)
     plt.ylim(0, 3.5)
-    plt.xticks([])
+    plt.xticks(fontsize=FONTSIZE)
-    plt.yticks([])
+    plt.yticks(fontsize=FONTSIZE)
     plt.xlabel('X [m]')
     plt.ylabel('Z [m]')
     if save:
         path_fig = os.path.join(dir_out, 'output_method.png')
-        plt.savefig(path_fig)
+        plt.savefig(path_fig, dpi=DPI)
         print("Figure of method saved in {}".format(path_fig))
+    plt.close('all')
+
+
+def show_box_plot(dic_errors, clusters, dir_fig, show=False, save=False):
+    excl_clusters = ['all', 'easy', 'moderate', 'hard']
+    clusters = [int(clst) for clst in clusters if clst not in excl_clusters]
+    methods = ('monstereo', 'pseudo-lidar', '3dop', 'monoloco')
+    y_min = 0
+    y_max = 16  # 18 for the other
+    xxs = get_distances(clusters)
+    labels = [str(xx) for xx in xxs]
+    for idx, method in enumerate(methods):
+        df = DATAFRAME([dic_errors[method][str(clst)] for clst in clusters[:-1]]).T
+        df.columns = labels
+
+        plt.figure(idx, figsize=FIGSIZE)  # with 200 dpi it becomes 1920x1440
+        _ = df.boxplot()
+        name = 'MonStereo' if method == 'monstereo' else method
+        plt.title(name, fontsize=FONTSIZE)
+        plt.ylabel('Average localization error (ALE) [m]', fontsize=FONTSIZE)
+        plt.xlabel('Ground-truth distance [m]', fontsize=FONTSIZE)
+        plt.xticks(fontsize=FONTSIZE)
+        plt.yticks(fontsize=FONTSIZE)
+        plt.ylim(y_min, y_max)
+
+        if save:
+            path_fig = os.path.join(dir_fig, 'box_plot_' + name + '.png')
+            plt.tight_layout()
+            plt.savefig(path_fig, dpi=DPI)
+            print("Figure of box plot saved in {}".format(path_fig))
+        if show:
+            plt.show()
+        plt.close('all')
 
 
 def target_error(xx, mm):
@@ -203,13 +249,10 @@ def get_confidence(xx, zz, std):
 
 def get_distances(clusters):
     """Extract distances as intermediate values between 2 clusters"""
-
-    clusters_ext = list(clusters)
-    clusters_ext.insert(0, str(0))
     distances = []
-    for idx, _ in enumerate(clusters_ext[:-1]):
+    for idx, _ in enumerate(clusters[:-1]):
-        clst_0 = float(clusters_ext[idx])
+        clst_0 = float(clusters[idx])
-        clst_1 = float(clusters_ext[idx + 1])
+        clst_1 = float(clusters[idx + 1])
         distances.append((clst_1 - clst_0) / 2 + clst_0)
     return tuple(distances)
 
@@ -251,49 +294,33 @@ def expandgrid(*itrs):
     return combinations
 
 
-def plot_dist(dist_gmm, dist_men, dist_women):
+# def get_percentile(dist_gmm):
-    try:
+#     dd_gt = 1000
-        import seaborn as sns  # pylint: disable=C0415
+#     mu_gmm = np.mean(dist_gmm)
-        sns.distplot(dist_men, hist=False, rug=False, label="Men")
+#     dist_d = dd_gt * mu_gmm / dist_gmm
-        sns.distplot(dist_women, hist=False, rug=False, label="Women")
+#     perc_d, _ = np.nanpercentile(dist_d, [18.5, 81.5])  # Laplace bi => 63%
-        sns.distplot(dist_gmm, hist=False, rug=False, label="GMM")
+#     perc_d2, _ = np.nanpercentile(dist_d, [23, 77])
-        plt.xlabel("X [cm]")
+#     mu_d = np.mean(dist_d)
-        plt.ylabel("Height distributions of men and women")
+#     # mm_bi = (mu_d - perc_d) / mu_d
-        plt.legend()
+#     # mm_test = (mu_d - perc_d2) / mu_d
-        plt.show()
+#     # mad_d = np.mean(np.abs(dist_d - mu_d))
-        plt.close()
-    except ImportError:
-        print("Import Seaborn first")
 
 
-def get_percentile(dist_gmm):
+def printing_styles(net):
-    dd_gt = 1000
+    if net == 'monstereo':
-    mu_gmm = np.mean(dist_gmm)
+        style = {"labels": ['3DOP', 'PSF', 'MonoLoco', 'MonoPSR', 'Pseudo-Lidar', 'Our MonStereo'],
-    dist_d = dd_gt * mu_gmm / dist_gmm
+                 "methods": ['3dop', 'psf', 'monoloco', 'monopsr', 'pseudo-lidar', 'monstereo'],
-    perc_d, _ = np.nanpercentile(dist_d, [18.5, 81.5])  # Laplace bi => 63%
+                 "mks": ['s', 'p', 'o', 'v', '*', '^'],
-    perc_d2, _ = np.nanpercentile(dist_d, [23, 77])
+                 "mksizes": [6, 6, 6, 6, 6, 6], "lws": [2, 2, 2, 2, 2, 2.2],
-    mu_d = np.mean(dist_d)
+                 "colors": ['gold', 'skyblue', 'darkgreen', 'pink', 'darkorange', 'b'],
-    # mm_bi = (mu_d - perc_d) / mu_d
+                 "lstyles": ['solid', 'solid', 'dashed', 'dashed', 'solid', 'solid']}
-    # mm_test = (mu_d - perc_d2) / mu_d
+    else:
-    # mad_d = np.mean(np.abs(dist_d - mu_d))
+        style = {"labels": ['Geometric Baseline', 'MonoPSR', 'MonoDIS', '3DOP (stereo)',
-
+                            'MonoLoco', 'Monoloco++'],
-
+                 "methods": ['geometric', 'monopsr', 'monodis', '3dop', 'monoloco', 'monoloco_pp'],
-def printing_styles(stereo):
+                 "mks": ['*', '^', 'p', '.', 's', 'o', 'o'],
-    style = {'mono': {"labels": ['Mono3D', 'Geometric Baseline', 'MonoDepth', 'Our MonoLoco', '3DOP (stereo)'],
+                 "mksizes": [6, 6, 6, 6, 6, 6], "lws": [1.5, 1.5, 1.5, 1.5, 1.5, 2.2],
-                      "methods": ['m3d_merged', 'geometric_merged', 'monodepth_merged', 'monoloco_merged',
+                 "colors": ['purple', 'olive', 'r', 'darkorange', 'b', 'darkblue'],
-                                  '3dop_merged'],
+                 "lstyles": ['solid', 'solid', 'solid', 'dashdot', 'solid', 'solid', ]}
-                      "mks": ['*', '^', 'p', 's', 'o'],
-                      "mksizes": [6, 6, 6, 6, 6], "lws": [1.5, 1.5, 1.5, 2.2, 1.6],
-                      "colors": ['r', 'deepskyblue', 'grey', 'b', 'darkorange'],
-                      "lstyles": ['solid', 'solid', 'solid', 'solid', 'dashdot']}}
-    if stereo:
-        style['stereo'] = {"labels": ['3DOP', 'Pose Baseline', 'ReiD Baseline', 'Our MonoLoco (monocular)',
-                                      'Our Stereo Baseline'],
-                           "methods": ['3dop_merged', 'pose_merged', 'reid_merged', 'monoloco_merged',
-                                       'ml_stereo_merged'],
-                           "mks": ['o', '^', 'p', 's', 's'],
-                           "mksizes": [6, 6, 6, 4, 6], "lws": [1.5, 1.5, 1.5, 1.2, 1.5],
-                           "colors": ['darkorange', 'lightblue', 'red', 'b', 'b'],
-                           "lstyles": ['solid', 'solid', 'solid', 'dashed', 'solid']}
 
     return style

monoloco/visuals/pifpaf_show.py

@@ -0,0 +1,460 @@
+
+"""
+Adapted from https://github.com/openpifpaf,
+which is: 'Copyright 2019-2021 by Sven Kreiss and contributors. All rights reserved.'
+and licensed under GNU AGPLv3
+"""
+
+from contextlib import contextmanager
+import math
+
+import numpy as np
+from PIL import Image
+
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.patches import Circle, FancyArrow
+import scipy.ndimage as ndimage
+
+
+COCO_PERSON_SKELETON = [
+    [16, 14], [14, 12], [17, 15], [15, 13], [12, 13], [6, 12], [7, 13],
+    [6, 7], [6, 8], [7, 9], [8, 10], [9, 11], [2, 3], [1, 2], [1, 3],
+    [2, 4], [3, 5], [4, 6], [5, 7]]
+
+
+@contextmanager
+def canvas(fig_file=None, show=True, **kwargs):
+    if 'figsize' not in kwargs:
+        # kwargs['figsize'] = (15, 8)
+        kwargs['figsize'] = (10, 6)
+    fig, ax = plt.subplots(**kwargs)
+
+    yield ax
+
+    fig.set_tight_layout(True)
+    if fig_file:
+        fig.savefig(fig_file, dpi=200)  # , bbox_inches='tight')
+    if show:
+        plt.show()
+    plt.close(fig)
+
+
+@contextmanager
+def image_canvas(image, fig_file=None, show=True, dpi_factor=1.0, fig_width=10.0, **kwargs):
+    if 'figsize' not in kwargs:
+        kwargs['figsize'] = (fig_width, fig_width * image.size[1] / image.size[0])
+
+    if plt is None:
+        raise Exception('please install matplotlib')
+    if ndimage is None:
+        raise Exception('please install scipy')
+    fig = plt.figure(**kwargs)
+    ax = plt.Axes(fig, [0.0, 0.0, 1.0, 1.0])
+    ax.set_axis_off()
+    ax.set_xlim(0, image.size[0])
+    ax.set_ylim(image.size[1], 0)
+    fig.add_axes(ax)
+    image_2 = ndimage.gaussian_filter(image, sigma=2.5)
+    ax.imshow(image_2, alpha=0.4)
+    yield ax
+
+    if fig_file:
+        fig.savefig(fig_file, dpi=image.size[0] / kwargs['figsize'][0] * dpi_factor)
+        print('keypoints image saved')
+    if show:
+        plt.show()
+    plt.close(fig)
+
+
+def load_image(path, scale=1.0):
+    with open(path, 'rb') as f:
+        image = Image.open(f).convert('RGB')
+        image = np.asarray(image) * scale / 255.0
+        return image
+
+
+def highlighted_arm(x, y, connection, color, lwidth, raise_hand, size=None):
+
+    c = color
+    linewidth = lwidth
+
+    width, height = (1,1)
+    if size:
+        width = size[0]
+        height = size[1]
+
+    l_arm_width = np.sqrt(((x[9]-x[7])/width)**2 + ((y[9]-y[7])/height)**2)*100
+    r_arm_width = np.sqrt(((x[10]-x[8])/width)**2 + ((y[10]-y[8])/height)**2)*100
+
+    if ((connection[0] == 5 and connection[1] == 7)
+        or (connection[0] == 7 and connection[1] == 9)) and raise_hand in ['left','both']:
+        c = 'yellow'
+        linewidth = l_arm_width
+    if ((connection[0] == 6 and connection[1] == 8)
+        or (connection[0] == 8 and connection[1] == 10)) and raise_hand in ['right', 'both']:
+        c = 'yellow'
+        linewidth = r_arm_width
+
+    return c, linewidth
+
+
+class KeypointPainter:
+    def __init__(self, *,
+                 skeleton=None,
+                 xy_scale=1.0, y_scale=1.0, highlight=None, highlight_invisible=False,
+                 show_box=True, linewidth=2, markersize=3,
+                 color_connections=False,
+                 solid_threshold=0.5):
+        self.skeleton = skeleton or COCO_PERSON_SKELETON
+        self.xy_scale = xy_scale
+        self.y_scale = y_scale
+        self.highlight = highlight
+        self.highlight_invisible = highlight_invisible
+        self.show_box = show_box
+        self.linewidth = linewidth
+        self.markersize = markersize
+        self.color_connections = color_connections
+        self.solid_threshold = solid_threshold
+        self.dashed_threshold = 0.1  # Patch to still allow force complete pose (set to zero to resume original)
+
+
+    def _draw_skeleton(self, ax, x, y, v, *, i=0, size=None, color=None, activities=None, dic_out=None):
+        if not np.any(v > 0):
+            return
+
+        if self.skeleton is not None:
+            for ci, connection in enumerate(np.array(self.skeleton) - 1):
+                c = color
+                linewidth = self.linewidth
+
+                if 'raise_hand' in activities:
+                    c, linewidth = highlighted_arm(x, y, connection, c, linewidth,
+                                                            dic_out['raising_hand'][:][i], size=size)
+                if 'is_turning' in activities:
+                    c, linewidth = highlighted_arm(x, y, connection, c, linewidth,
+                                                            dic_out['turning'][:][i], size=size)
+
+                if self.color_connections:
+                    c = matplotlib.cm.get_cmap('tab20')(ci / len(self.skeleton))
+                if np.all(v[connection] > self.dashed_threshold):
+                    ax.plot(x[connection], y[connection],
+                            linewidth=linewidth, color=c,
+                            linestyle='dashed', dash_capstyle='round')
+                if np.all(v[connection] > self.solid_threshold):
+                    ax.plot(x[connection], y[connection],
+                            linewidth=linewidth, color=c, solid_capstyle='round')
+
+        # highlight invisible keypoints
+        inv_color = 'k' if self.highlight_invisible else color
+
+        ax.plot(x[v > self.dashed_threshold], y[v > self.dashed_threshold],
+                'o', markersize=self.markersize,
+                markerfacecolor=color, markeredgecolor=inv_color, markeredgewidth=2)
+        ax.plot(x[v > self.solid_threshold], y[v > self.solid_threshold],
+                'o', markersize=self.markersize,
+                markerfacecolor=color, markeredgecolor=color, markeredgewidth=2)
+
+        if self.highlight is not None:
+            v_highlight = v[self.highlight]
+            ax.plot(x[self.highlight][v_highlight > 0],
+                    y[self.highlight][v_highlight > 0],
+                    'o', markersize=self.markersize*2, markeredgewidth=2,
+                    markerfacecolor=color, markeredgecolor=color)
+
+    @staticmethod
+    def _draw_box(ax, x, y, v, color, score=None):
+        if not np.any(v > 0):
+            return
+
+        # keypoint bounding box
+        x1, x2 = np.min(x[v > 0]), np.max(x[v > 0])
+        y1, y2 = np.min(y[v > 0]), np.max(y[v > 0])
+        if x2 - x1 < 5.0:
+            x1 -= 2.0
+            x2 += 2.0
+        if y2 - y1 < 5.0:
+            y1 -= 2.0
+            y2 += 2.0
+        ax.add_patch(
+            matplotlib.patches.Rectangle(
+                (x1, y1), x2 - x1, y2 - y1, fill=False, color=color))
+
+        if score:
+            ax.text(x1, y1, '{:.4f}'.format(score), fontsize=8, color=color)
+
+    @staticmethod
+    def _draw_text(ax, x, y, v, text, color, fontsize=8):
+        if not np.any(v > 0):
+            return
+
+        # keypoint bounding box
+        x1, x2 = np.min(x[v > 0]), np.max(x[v > 0])
+        y1, y2 = np.min(y[v > 0]), np.max(y[v > 0])
+        if x2 - x1 < 5.0:
+            x1 -= 2.0
+            x2 += 2.0
+        if y2 - y1 < 5.0:
+            y1 -= 2.0
+            y2 += 2.0
+
+        ax.text(x1 + 2, y1 - 2, text, fontsize=fontsize,
+                color='white', bbox={'facecolor': color, 'alpha': 0.5, 'linewidth': 0})
+
+    @staticmethod
+    def _draw_scales(ax, xs, ys, vs, color, scales):
+        for x, y, v, scale in zip(xs, ys, vs, scales):
+            if v == 0.0:
+                continue
+            ax.add_patch(
+                matplotlib.patches.Rectangle(
+                    (x - scale, y - scale), 2 * scale, 2 * scale, fill=False, color=color))
+
+    def keypoints(self, ax, keypoint_sets, *,
+                  size=None, scores=None, color=None,
+                  colors=None, texts=None, activities=None, dic_out=None):
+        if keypoint_sets is None:
+            return
+
+        if color is None and self.color_connections:
+            color = 'white'
+        if color is None and colors is None:
+            colors = range(len(keypoint_sets))
+
+        for i, kps in enumerate(np.asarray(keypoint_sets)):
+            assert kps.shape[1] == 3
+            x = kps[:, 0] * self.xy_scale
+            y = kps[:, 1] * self.xy_scale * self.y_scale
+            v = kps[:, 2]
+
+            if colors is not None:
+                color = colors[i]
+
+            if isinstance(color, (int, np.integer)):
+                color = matplotlib.cm.get_cmap('tab20')((color % 20 + 0.05) / 20)
+
+            self._draw_skeleton(ax, x, y, v, i=i, size=size, color=color, activities=activities, dic_out=dic_out)
+
+            score = scores[i] if scores is not None else None
+            if score is not None:
+                z_str = str(score).split(sep='.')
+                text = z_str[0] + '.' + z_str[1][0]
+                self._draw_text(ax, x[1:3], y[1:3]-5, v[1:3], text, color, fontsize=16)
+            if self.show_box:
+                score = scores[i] if scores is not None else None
+                self._draw_box(ax, x, y, v, color, score)
+
+                if texts is not None:
+                    self._draw_text(ax, x, y, v, texts[i], color)
+
+
+    def annotations(self, ax, annotations, *,
+                    color=None, colors=None, texts=None):
+        if annotations is None:
+            return
+
+        if color is None and self.color_connections:
+            color = 'white'
+        if color is None and colors is None:
+            colors = range(len(annotations))
+
+        for i, ann in enumerate(annotations):
+            if colors is not None:
+                color = colors[i]
+
+            text = texts[i] if texts is not None else None
+            self.annotation(ax, ann, color=color, text=text)
+
+    def annotation(self, ax, ann, *, color, text=None):
+        if isinstance(color, (int, np.integer)):
+            color = matplotlib.cm.get_cmap('tab20')((color % 20 + 0.05) / 20)
+
+        kps = ann.data
+        assert kps.shape[1] == 3
+        x = kps[:, 0] * self.xy_scale
+        y = kps[:, 1] * self.xy_scale
+        v = kps[:, 2]
+
+        self._draw_skeleton(ax, x, y, v, color=color)
+
+        if ann.joint_scales is not None:
+            self._draw_scales(ax, x, y, v, color, ann.joint_scales)
+
+        if self.show_box:
+            self._draw_box(ax, x, y, v, color, ann.score())
+
+            if text is not None:
+                self._draw_text(ax, x, y, v, text, color)
+
+
+def quiver(ax, vector_field, intensity_field=None, step=1, threshold=0.5,
+           xy_scale=1.0, uv_is_offset=False,
+           reg_uncertainty=None, **kwargs):
+    x, y, u, v, c, r = [], [], [], [], [], []
+    for j in range(0, vector_field.shape[1], step):
+        for i in range(0, vector_field.shape[2], step):
+            if intensity_field is not None and intensity_field[j, i] < threshold:
+                continue
+            x.append(i * xy_scale)
+            y.append(j * xy_scale)
+            u.append(vector_field[0, j, i] * xy_scale)
+            v.append(vector_field[1, j, i] * xy_scale)
+            c.append(intensity_field[j, i] if intensity_field is not None else 1.0)
+            r.append(reg_uncertainty[j, i] * xy_scale if reg_uncertainty is not None else None)
+    x = np.array(x)
+    y = np.array(y)
+    u = np.array(u)
+    v = np.array(v)
+    c = np.array(c)
+    r = np.array(r)
+    s = np.argsort(c)
+    if uv_is_offset:
+        u -= x
+        v -= y
+
+    for xx, yy, uu, vv, _, rr in zip(x, y, u, v, c, r):
+        if not rr:
+            continue
+        circle = matplotlib.patches.Circle(
+            (xx + uu, yy + vv), rr / 2.0, zorder=11, linewidth=1, alpha=1.0,
+            fill=False, color='orange')
+        ax.add_artist(circle)
+
+    return ax.quiver(x[s], y[s], u[s], v[s], c[s],
+                     angles='xy', scale_units='xy', scale=1, zOrder=10, **kwargs)
+
+
+def arrows(ax, fourd, xy_scale=1.0, threshold=0.0, **kwargs):
+    mask = np.min(fourd[:, 2], axis=0) >= threshold
+    fourd = fourd[:, :, mask]
+    (x1, y1), (x2, y2) = fourd[:, :2, :] * xy_scale
+    c = np.min(fourd[:, 2], axis=0)
+    s = np.argsort(c)
+    return ax.quiver(x1[s], y1[s], (x2 - x1)[s], (y2 - y1)[s], c[s],
+                     angles='xy', scale_units='xy', scale=1, zOrder=10, **kwargs)
+
+
+def boxes(ax, scalar_field, intensity_field=None, xy_scale=1.0, step=1, threshold=0.5,
+          cmap='viridis_r', clim=(0.5, 1.0), **kwargs):
+    x, y, s, c = [], [], [], []
+    for j in range(0, scalar_field.shape[0], step):
+        for i in range(0, scalar_field.shape[1], step):
+            if intensity_field is not None and intensity_field[j, i] < threshold:
+                continue
+            x.append(i * xy_scale)
+            y.append(j * xy_scale)
+            s.append(scalar_field[j, i] * xy_scale)
+            c.append(intensity_field[j, i] if intensity_field is not None else 1.0)
+
+    cmap = matplotlib.cm.get_cmap(cmap)
+    cnorm = matplotlib.colors.Normalize(vmin=clim[0], vmax=clim[1])
+    for xx, yy, ss, cc in zip(x, y, s, c):
+        color = cmap(cnorm(cc))
+        rectangle = matplotlib.patches.Rectangle(
+            (xx - ss, yy - ss), ss * 2.0, ss * 2.0,
+            color=color, zorder=10, linewidth=1, **kwargs)
+        ax.add_artist(rectangle)
+
+
+def circles(ax, scalar_field, intensity_field=None, xy_scale=1.0, step=1, threshold=0.5,
+            cmap='viridis_r', clim=(0.5, 1.0), **kwargs):
+    x, y, s, c = [], [], [], []
+    for j in range(0, scalar_field.shape[0], step):
+        for i in range(0, scalar_field.shape[1], step):
+            if intensity_field is not None and intensity_field[j, i] < threshold:
+                continue
+            x.append(i * xy_scale)
+            y.append(j * xy_scale)
+            s.append(scalar_field[j, i] * xy_scale)
+            c.append(intensity_field[j, i] if intensity_field is not None else 1.0)
+
+    cmap = matplotlib.cm.get_cmap(cmap)
+    cnorm = matplotlib.colors.Normalize(vmin=clim[0], vmax=clim[1])
+    for xx, yy, ss, cc in zip(x, y, s, c):
+        color = cmap(cnorm(cc))
+        circle = matplotlib.patches.Circle(
+            (xx, yy), ss,
+            color=color, zorder=10, linewidth=1, **kwargs)
+        ax.add_artist(circle)
+
+
+def white_screen(ax, alpha=0.9):
+    ax.add_patch(
+        plt.Rectangle((0, 0), 1, 1, transform=ax.transAxes, alpha=alpha,
+                      facecolor='white')
+    )
+
+
+def get_pifpaf_outputs(annotations):
+    # TODO extract direct from predictions with pifpaf 0.11+
+    """Extract keypoints sets and scores from output dictionary"""
+    if not annotations:
+        return [], []
+    keypoints_sets = np.array([dic['keypoints']
+                               for dic in annotations]).reshape((-1, 17, 3))
+    score_weights = np.ones((keypoints_sets.shape[0], 17))
+    score_weights[:, 3] = 3.0
+    score_weights /= np.sum(score_weights[0, :])
+    kps_scores = keypoints_sets[:, :, 2]
+    ordered_kps_scores = np.sort(kps_scores, axis=1)[:, ::-1]
+    scores = np.sum(score_weights * ordered_kps_scores, axis=1)
+    return keypoints_sets, scores
+
+
+def draw_orientation(ax, centers, sizes, angles, colors, mode):
+
+    if mode == 'front':
+        length = 5
+        fill = False
+        alpha = 0.6
+        zorder_circle = 0.5
+        zorder_arrow = 5
+        linewidth = 1.5
+        edgecolor = 'k'
+        radiuses = [s / 1.2 for s in sizes]
+    else:
+        length = 1.3
+        head_width = 0.3
+        linewidth = 2
+        radiuses = [0.2] * len(centers)
+        # length = 1.6
+        # head_width = 0.4
+        # linewidth = 2.7
+        radiuses = [0.2] * len(centers)
+        fill = True
+        alpha = 1
+        zorder_circle = 2
+        zorder_arrow = 1
+
+    for idx, theta in enumerate(angles):
+        color = colors[idx]
+        radius = radiuses[idx]
+
+        if mode == 'front':
+            x_arr = centers[idx][0] + (length + radius) * math.cos(theta)
+            z_arr = length + centers[idx][1] + \
+                (length + radius) * math.sin(theta)
+            delta_x = math.cos(theta)
+            delta_z = math.sin(theta)
+            head_width = max(10, radiuses[idx] / 1.5)
+
+        else:
+            edgecolor = color
+            x_arr = centers[idx][0]
+            z_arr = centers[idx][1]
+            delta_x = length * math.cos(theta)
+            # keep into account kitti convention
+            delta_z = - length * math.sin(theta)
+
+        circle = Circle(centers[idx], radius=radius, color=color,
+                        fill=fill, alpha=alpha, zorder=zorder_circle)
+        arrow = FancyArrow(x_arr, z_arr, delta_x, delta_z, head_width=head_width, edgecolor=edgecolor,
+                           facecolor=color, linewidth=linewidth, zorder=zorder_arrow)
+        ax.add_patch(circle)
+        ax.add_patch(arrow)
+
+
+def social_distance_colors(colors, dic_out):
+    # Prepare color for social distancing
+    colors = ['r' if flag else colors[idx] for idx,flag in enumerate(dic_out['social_distance'])]
+    return colors

monoloco/visuals/plot_3d_box.py

@@ -0,0 +1,95 @@
+
+import numpy as np
+
+
+def correct_boxes(boxes, hwls, xyzs, yaws, path_calib):
+
+    with open(path_calib, "r") as ff:
+        file = ff.readlines()
+    p2_str = file[2].split()[1:]
+    p2_list = [float(xx) for xx in p2_str]
+    P = np.array(p2_list).reshape(3, 4)
+    boxes_new = []
+    for idx in range(boxes):
+        hwl = hwls[idx]
+        xyz = xyzs[idx]
+        yaw = yaws[idx]
+        corners_2d, _ = compute_box_3d(hwl, xyz, yaw, P)
+        box_new = project_8p_to_4p(corners_2d).reshape(-1).tolist()
+        boxes_new.append(box_new)
+    return boxes_new
+
+
+def compute_box_3d(hwl, xyz, ry, P):
+    """ Takes an object and a projection matrix (P) and projects the 3d
+        bounding box into the image plane.
+        Returns:
+            corners_2d: (8,2) array in left image coord.
+            corners_3d: (8,3) array in in rect camera coord.
+    """
+    # compute rotational matrix around yaw axis
+    R = roty(ry)
+
+    # 3d bounding box dimensions
+    l = hwl[2]
+    w = hwl[1]
+    h = hwl[0]
+
+    # 3d bounding box corners
+    x_corners = [l / 2, l / 2, -l / 2, -l / 2, l / 2, l / 2, -l / 2, -l / 2]
+    y_corners = [0, 0, 0, 0, -h, -h, -h, -h]
+    z_corners = [w / 2, -w / 2, -w / 2, w / 2, w / 2, -w / 2, -w / 2, w / 2]
+
+    # rotate and translate 3d bounding box
+    corners_3d = np.dot(R, np.vstack([x_corners, y_corners, z_corners]))
+    # print corners_3d.shape
+    corners_3d[0, :] = corners_3d[0, :] + xyz[0]
+    corners_3d[1, :] = corners_3d[1, :] + xyz[1]
+    corners_3d[2, :] = corners_3d[2, :] + xyz[2]
+    # print 'cornsers_3d: ', corners_3d
+    # only draw 3d bounding box for objs in front of the camera
+    if np.any(corners_3d[2, :] < 0.1):
+        corners_2d = None
+        return corners_2d, np.transpose(corners_3d)
+
+    # project the 3d bounding box into the image plane
+    corners_2d = project_to_image(np.transpose(corners_3d), P)
+    # print 'corners_2d: ', corners_2d
+    return corners_2d, np.transpose(corners_3d)
+
+
+def roty(t):
+    """ Rotation about the y-axis. """
+    c = np.cos(t)
+    s = np.sin(t)
+    return np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]])
+
+
+def project_to_image(pts_3d, P):
+    """ Project 3d points to image plane.
+    Usage: pts_2d = projectToImage(pts_3d, P)
+      input: pts_3d: nx3 matrix
+             P:      3x4 projection matrix
+      output: pts_2d: nx2 matrix
+      P(3x4) dot pts_3d_extended(4xn) = projected_pts_2d(3xn)
+      => normalize projected_pts_2d(2xn)
+      <=> pts_3d_extended(nx4) dot P'(4x3) = projected_pts_2d(nx3)
+          => normalize projected_pts_2d(nx2)
+    """
+    n = pts_3d.shape[0]
+    pts_3d_extend = np.hstack((pts_3d, np.ones((n, 1))))
+    # print(('pts_3d_extend shape: ', pts_3d_extend.shape))
+    pts_2d = np.dot(pts_3d_extend, np.transpose(P))  # nx3
+    pts_2d[:, 0] /= pts_2d[:, 2]
+    pts_2d[:, 1] /= pts_2d[:, 2]
+    return pts_2d[:, 0:2]
+
+
+def project_8p_to_4p(pts_2d):
+    x0 = np.min(pts_2d[:, 0])
+    x1 = np.max(pts_2d[:, 0])
+    y0 = np.min(pts_2d[:, 1])
+    y1 = np.max(pts_2d[:, 1])
+    x0 = max(0, x0)
+    y0 = max(0, y0)
+    return np.array([x0, y0, x1, y1])

monoloco/visuals/printer.py

@@ -1,103 +1,162 @@
-
+"""
+Class for drawing frontal, bird-eye-view and multi figures
+"""
+# pylint: disable=attribute-defined-outside-init
 import math
 from collections import OrderedDict
 
-import numpy as np
-import matplotlib
 import matplotlib.pyplot as plt
-import matplotlib.cm as cm
+from matplotlib.patches import Rectangle
-from matplotlib.patches import Ellipse, Circle, Rectangle
-from mpl_toolkits.axes_grid1 import make_axes_locatable
 
-from ..utils import pixel_to_camera, get_task_error
+from .pifpaf_show import KeypointPainter, get_pifpaf_outputs, draw_orientation, social_distance_colors
+from ..utils import pixel_to_camera
+
+
+def get_angle(xx, zz):
+    """Obtain the points to plot the confidence of each annotation"""
+
+    theta = math.atan2(zz, xx)
+    angle = theta * (180 / math.pi)
+
+    return angle
+
+
+def image_attributes(dpi, output_types):
+    c = 0.7 if 'front' in output_types else 1.0
+    return dict(dpi=dpi,
+                fontsize_d=round(14 * c),
+                fontsize_bv=round(24 * c),
+                fontsize_num=round(22 * c),
+                fontsize_ax=round(16 * c),
+                linewidth=round(8 * c),
+                markersize=round(13 * c),
+                y_box_margin=round(24 * math.sqrt(c)),
+                stereo=dict(color='deepskyblue',
+                            numcolor='darkorange',
+                            linewidth=1 * c),
+                mono=dict(color='red',
+                          numcolor='firebrick',
+                          linewidth=2 * c)
+                )
 
 
 class Printer:
     """
     Print results on images: birds eye view and computed distance
     """
-    FONTSIZE_BV = 16
+    FIG_WIDTH = 15
-    FONTSIZE = 18
+    extensions = []
-    TEXTCOLOR = 'darkorange'
+    y_scale = 1
-    COLOR_KPS = 'yellow'
+    nones = lambda n: [None for _ in range(n)]
-
+    mpl_im0, stds_ale, stds_epi, xx_gt, zz_gt, xx_pred, zz_pred, dd_real, uv_centers, uv_shoulders, uv_kps, boxes, \
-    def __init__(self, image, output_path, kk, output_types, epistemic=False, z_max=30, fig_width=10):
+        boxes_gt, uv_camera, radius, auxs = nones(16)
 
+    def __init__(self, image, output_path, kk, args):
         self.im = image
-        self.kk = kk
-        self.output_types = output_types
-        self.epistemic = epistemic
-        self.z_max = z_max  # To include ellipses in the image
-        self.y_scale = 1
         self.width = self.im.size[0]
         self.height = self.im.size[1]
-        self.fig_width = fig_width
-
-        # Define the output dir
         self.output_path = output_path
-        self.cmap = cm.get_cmap('jet')
+        self.kk = kk
-        self.extensions = []
+        self.output_types = args.output_types
+        self.z_max = args.z_max  # set max distance to show instances
+        self.webcam = args.webcam
+        self.show_all = args.show_all or self.webcam
+        self.show = args.show_all or self.webcam
+        self.save = not args.no_save and not self.webcam
+        self.plt_close = not self.webcam
+        self.activities = args.activities
+        self.hide_distance = args.hide_distance
 
-        # Define variables of the class to change for every image
+        # define image attributes
-        self.mpl_im0 = self.stds_ale = self.stds_epi = self.xx_gt = self.zz_gt = self.xx_pred = self.zz_pred =\
+        self.attr = image_attributes(args.dpi, args.output_types)
-            self.dds_real = self.uv_centers = self.uv_shoulders = self.uv_kps = self.boxes = self.boxes_gt = \
-            self.uv_camera = self.radius = None
 
     def _process_results(self, dic_ann):
         # Include the vectors inside the interval given by z_max
+        self.angles = dic_ann['angles']
         self.stds_ale = dic_ann['stds_ale']
         self.stds_epi = dic_ann['stds_epi']
+        self.gt = dic_ann['gt']  # regulate ground-truth matching
         self.xx_gt = [xx[0] for xx in dic_ann['xyz_real']]
+        self.xx_pred = [xx[0] for xx in dic_ann['xyz_pred']]
+
+        self.xz_centers = [[xx[0], xx[2]] for xx in dic_ann['xyz_pred']]
+        # Set maximum distance
+        self.dd_pred = dic_ann['dds_pred']
+        self.dd_real = dic_ann['dds_real']
+        self.z_max = int(min(self.z_max, 4 + max(max(self.dd_pred), max(self.dd_real, default=0))))
+
+        # Do not print instances outside z_max
         self.zz_gt = [xx[2] if xx[2] < self.z_max - self.stds_epi[idx] else 0
                       for idx, xx in enumerate(dic_ann['xyz_real'])]
-        self.xx_pred = [xx[0] for xx in dic_ann['xyz_pred']]
         self.zz_pred = [xx[2] if xx[2] < self.z_max - self.stds_epi[idx] else 0
                         for idx, xx in enumerate(dic_ann['xyz_pred'])]
-        self.dds_real = dic_ann['dds_real']
+
+        self.uv_heads = dic_ann['uv_heads']
+        self.centers = self.uv_heads
+        if 'multi' in self.output_types:
+            for center in self.centers:
+                center[1] = center[1] * self.y_scale
         self.uv_shoulders = dic_ann['uv_shoulders']
         self.boxes = dic_ann['boxes']
         self.boxes_gt = dic_ann['boxes_gt']
-
         self.uv_camera = (int(self.im.size[0] / 2), self.im.size[1])
-        self.radius = 11 / 1600 * self.width
+        self.auxs = dic_ann['aux']
+        if len(self.auxs) == 0:
+            self.modes = ['mono'] * len(self.dd_pred)
+        else:
+            self.modes = []
+            for aux in self.auxs:
+                if aux <= 0.3:
+                    self.modes.append('mono')
+                else:
+                    self.modes.append('stereo')
+
+    def factory_axes(self, dic_out):
+        """Create axes for figures: front bird multi"""
+
+        if self.webcam:
+            plt.style.use('dark_background')
 
-    def factory_axes(self):
-        """Create axes for figures: front bird combined"""
         axes = []
         figures = []
 
-        #  Initialize combined figure, resizing it for aesthetic proportions
+        # Process the annotation dictionary of monoloco
-        if 'combined' in self.output_types:
+        if dic_out:
-            assert 'bird' and 'front' not in self.output_types, \
+            self._process_results(dic_out)
-                "combined figure cannot be print together with front or bird ones"
 
-            self.y_scale = self.width / (self.height * 1.8)  # Defined proportion
+        #  Initialize multi figure, resizing it for aesthetic proportion
+        if 'multi' in self.output_types:
+            assert 'bird' not in self.output_types and 'front' not in self.output_types, \
+                "multi figure cannot be print together with front or bird ones"
+
+            self.y_scale = self.width / (self.height * 2)  # Defined proportion
             if self.y_scale < 0.95 or self.y_scale > 1.05:  # allows more variation without resizing
                 self.im = self.im.resize((self.width, round(self.height * self.y_scale)))
             self.width = self.im.size[0]
             self.height = self.im.size[1]
-            fig_width = self.fig_width + 0.6 * self.fig_width
+            fig_width = self.FIG_WIDTH + 0.6 * self.FIG_WIDTH
-            fig_height = self.fig_width * self.height / self.width
+            fig_height = self.FIG_WIDTH * self.height / self.width
 
             # Distinguish between KITTI images and general images
-            fig_ar_1 = 1.7 if self.y_scale > 1.7 else 1.3
+            fig_ar_1 = 0.8
             width_ratio = 1.9
-            self.extensions.append('.combined.png')
+            self.extensions.append('.multi.png')
 
-            fig, (ax1, ax0) = plt.subplots(1, 2, sharey=False, gridspec_kw={'width_ratios': [1, width_ratio]},
+            fig, (ax0, ax1) = plt.subplots(1, 2, sharey=False, gridspec_kw={'width_ratios': [width_ratio, 1]},
                                            figsize=(fig_width, fig_height))
+
             ax1.set_aspect(fig_ar_1)
             fig.set_tight_layout(True)
             fig.subplots_adjust(left=0.02, right=0.98, bottom=0, top=1, hspace=0, wspace=0.02)
 
             figures.append(fig)
             assert 'front' not in self.output_types and 'bird' not in self.output_types, \
-                "--combined arguments is not supported with other visualizations"
+                "--multi arguments is not supported with other visualizations"
 
         # Initialize front figure
         elif 'front' in self.output_types:
-            width = self.fig_width
+            width = self.FIG_WIDTH
-            height = self.fig_width * self.height / self.width
+            height = self.FIG_WIDTH * self.height / self.width
             self.extensions.append(".front.png")
             plt.figure(0)
             fig0, ax0 = plt.subplots(1, 1, figsize=(width, height))
@@ -105,18 +164,8 @@ class Printer:
             figures.append(fig0)
 
         # Create front figure axis
-        if any(xx in self.output_types for xx in ['front', 'combined']):
+        if any(xx in self.output_types for xx in ['front', 'multi']):
-            ax0 = self.set_axes(ax0, axis=0)
+            ax0 = self._set_axes(ax0, axis=0)
-
-            divider = make_axes_locatable(ax0)
-            cax = divider.append_axes('right', size='3%', pad=0.05)
-            bar_ticks = self.z_max // 5 + 1
-            norm = matplotlib.colors.Normalize(vmin=0, vmax=self.z_max)
-            scalar_mappable = plt.cm.ScalarMappable(cmap=self.cmap, norm=norm)
-            scalar_mappable.set_array([])
-            plt.colorbar(scalar_mappable, ticks=np.linspace(0, self.z_max, bar_ticks),
-                         boundaries=np.arange(- 0.05, self.z_max + 0.1, .1), cax=cax, label='Z [m]')
-
             axes.append(ax0)
         if not axes:
             axes.append(None)
@@ -127,99 +176,139 @@ class Printer:
             fig1, ax1 = plt.subplots(1, 1)
             fig1.set_tight_layout(True)
             figures.append(fig1)
-        if any(xx in self.output_types for xx in ['bird', 'combined']):
+        if any(xx in self.output_types for xx in ['bird', 'multi']):
-            ax1 = self.set_axes(ax1, axis=1)  # Adding field of view
+            ax1 = self._set_axes(ax1, axis=1)  # Adding field of view
             axes.append(ax1)
         return figures, axes
 
-    def draw(self, figures, axes, dic_out, image, draw_text=True, legend=True, draw_box=False,
-             save=False, show=False):
 
-        # Process the annotation dictionary of monoloco
+    def _webcam_front(self, axis, colors, activities, annotations, dic_out):
-        self._process_results(dic_out)
+        sizes = [abs(self.centers[idx][1] - uv_s[1]*self.y_scale) / 1.5 for idx, uv_s in
+                 enumerate(self.uv_shoulders)]
 
-        # Draw the front figure
+        keypoint_sets, _ = get_pifpaf_outputs(annotations)
-        num = 0
+        keypoint_painter = KeypointPainter(show_box=False, y_scale=self.y_scale)
-        self.mpl_im0.set_data(image)
-        for idx, uv in enumerate(self.uv_shoulders):
-            if any(xx in self.output_types for xx in ['front', 'combined']) and \
-                 min(self.zz_pred[idx], self.zz_gt[idx]) > 0:
 
-                color = self.cmap((self.zz_pred[idx] % self.z_max) / self.z_max)
+        if not self.hide_distance:
-                self.draw_circle(axes, uv, color)
+            scores = self.dd_pred
-                if draw_box:
+        else:
-                    self.draw_boxes(axes, idx, color)
+            scores=None
 
-                if draw_text:
+        keypoint_painter.keypoints(
-                    self.draw_text_front(axes, uv, num)
+            axis, keypoint_sets, size=self.im.size,
-                    num += 1
+            scores=scores, colors=colors, activities=activities, dic_out=dic_out)
 
-        # Draw the bird figure
+        draw_orientation(axis, self.centers,
-        num = 0
+                        sizes, self.angles, colors, mode='front')
-        for idx, _ in enumerate(self.xx_pred):
-            if any(xx in self.output_types for xx in ['bird', 'combined']) and self.zz_gt[idx] > 0:
 
-                # Draw ground truth and predicted ellipses
+
-                self.draw_ellipses(axes, idx)
+    def _front_loop(self, iterator, axes, number, colors, annotations, dic_out):
+        for idx in iterator:
+            if any(xx in self.output_types for xx in ['front', 'multi']) and self.zz_pred[idx] > 0:
+                if self.webcam:
+                    self._webcam_front(axes[0], colors, self.activities, annotations, dic_out)
+                else:
+                    self._draw_front(axes[0],
+                                     self.dd_pred[idx],
+                                     idx,
+                                     number)
+                number['num'] += 1
+
+
+    def _bird_loop(self, iterator, axes, colors, number):
+        for idx in iterator:
+            if any(xx in self.output_types for xx in ['bird', 'multi']) and self.zz_pred[idx] > 0:
+                draw_orientation(axes[1], self.xz_centers[:len(iterator)], [],
+                                self.angles[:len(iterator)], colors, mode='bird')
+                # Draw ground truth and uncertainty
+                self._draw_uncertainty(axes, idx)
 
                 # Draw bird eye view text
-                if draw_text:
+                if number['flag']:
-                    self.draw_text_bird(axes, idx, num)
+                    self._draw_text_bird(axes, idx, number['num'])
-                    num += 1
+                    number['num'] += 1
-        # Add the legend
+
-        if legend:
+
-            draw_legend(axes)
+    def draw(self, figures, axes, image, dic_out=None, annotations=None):
+
+        colors = ['deepskyblue' for _ in self.uv_heads]
+        if 'social_distance' in self.activities:
+            colors = social_distance_colors(colors, dic_out)
+
+        # whether to include instances that don't match the ground-truth
+        iterator = range(len(self.zz_pred)) if self.show_all else range(len(self.zz_gt))
+        if not iterator:
+            print("-" * 110 + '\n' + '! No instances detected' '\n' + '-' * 110)
+
+        # Draw the front figure
+        number = dict(flag=False, num=97)
+        if any(xx in self.output_types for xx in ['front', 'multi']):
+            number['flag'] = True  # add numbers
+            # Remove image if social distance is activated
+            if 'social_distance' not in self.activities:
+                self.mpl_im0.set_data(image)
+
+        self._front_loop(iterator, axes, number, colors, annotations, dic_out)
+
+        # Draw the bird figure
+        number['num'] = 97
+        self._bird_loop(iterator, axes, colors, number)
+
+        self._draw_legend(axes)
 
         # Draw, save or/and show the figures
         for idx, fig in enumerate(figures):
             fig.canvas.draw()
-            if save:
+            if self.save:
-                fig.savefig(self.output_path + self.extensions[idx], bbox_inches='tight')
+                fig.savefig(self.output_path + self.extensions[idx], bbox_inches='tight', dpi=self.attr['dpi'])
-            if show:
+            if self.show:
                 fig.show()
+            if self.plt_close:
+                plt.close(fig)
 
-    def draw_ellipses(self, axes, idx):
-        """draw uncertainty ellipses"""
-        target = get_task_error(self.dds_real[idx])
-        angle_gt = get_angle(self.xx_gt[idx], self.zz_gt[idx])
-        ellipse_real = Ellipse((self.xx_gt[idx], self.zz_gt[idx]), width=target * 2, height=1,
-                               angle=angle_gt, color='lightgreen', fill=True, label="Task error")
-        axes[1].add_patch(ellipse_real)
-        if abs(self.zz_gt[idx] - self.zz_pred[idx]) > 0.001:
-            axes[1].plot(self.xx_gt[idx], self.zz_gt[idx], 'kx', label="Ground truth", markersize=3)
 
-        angle = get_angle(self.xx_pred[idx], self.zz_pred[idx])
+    def _draw_front(self, ax, z, idx, number):
-        ellipse_ale = Ellipse((self.xx_pred[idx], self.zz_pred[idx]), width=self.stds_ale[idx] * 2,
-                              height=1, angle=angle, color='b', fill=False, label="Aleatoric Uncertainty",
-                              linewidth=1.3)
-        ellipse_var = Ellipse((self.xx_pred[idx], self.zz_pred[idx]), width=self.stds_epi[idx] * 2,
-                              height=1, angle=angle, color='r', fill=False, label="Uncertainty",
-                              linewidth=1, linestyle='--')
 
-        axes[1].add_patch(ellipse_ale)
+        # Bbox
-        if self.epistemic:
+        w = min(self.width-2, self.boxes[idx][2] - self.boxes[idx][0])
-            axes[1].add_patch(ellipse_var)
+        h = min(self.height-2, (self.boxes[idx][3] - self.boxes[idx][1]) * self.y_scale)
+        x0 = self.boxes[idx][0]
+        y0 = self.boxes[idx][1] * self.y_scale
+        y1 = y0 + h
+        rectangle = Rectangle((x0, y0),
+                              width=w,
+                              height=h,
+                              fill=False,
+                              color=self.attr[self.modes[idx]]['color'],
+                              linewidth=self.attr[self.modes[idx]]['linewidth'])
+        ax.add_patch(rectangle)
+        z_str = str(z).split(sep='.')
+        text = z_str[0] + '.' + z_str[1][0]
+        bbox_config = {'facecolor': self.attr[self.modes[idx]]['color'], 'alpha': 0.4, 'linewidth': 0}
 
-        axes[1].plot(self.xx_pred[idx], self.zz_pred[idx], 'ro', label="Predicted", markersize=3)
+        x_t = x0 - 1.5
+        y_t = y1 + self.attr['y_box_margin']
+        if y_t < (self.height-10):
+            if not self.hide_distance:
+                ax.annotate(
+                    text,
+                    (x_t, y_t),
+                    fontsize=self.attr['fontsize_d'],
+                    weight='bold',
+                    xytext=(5.0, 5.0),
+                    textcoords='offset points',
+                    color='white',
+                    bbox=bbox_config,
+                )
+                if number['flag']:
+                    ax.text(x0 - 17,
+                            y1 + 14,
+                            chr(number['num']),
+                            fontsize=self.attr['fontsize_num'],
+                            color=self.attr[self.modes[idx]]['numcolor'],
+                            weight='bold')
 
-    def draw_boxes(self, axes, idx, color):
+    def _draw_text_bird(self, axes, idx, num):
-        ww_box = self.boxes[idx][2] - self.boxes[idx][0]
-        hh_box = (self.boxes[idx][3] - self.boxes[idx][1]) * self.y_scale
-        ww_box_gt = self.boxes_gt[idx][2] - self.boxes_gt[idx][0]
-        hh_box_gt = (self.boxes_gt[idx][3] - self.boxes_gt[idx][1]) * self.y_scale
-
-        rectangle = Rectangle((self.boxes[idx][0], self.boxes[idx][1] * self.y_scale),
-                              width=ww_box, height=hh_box, fill=False, color=color, linewidth=3)
-        rectangle_gt = Rectangle((self.boxes_gt[idx][0], self.boxes_gt[idx][1] * self.y_scale),
-                                 width=ww_box_gt, height=hh_box_gt, fill=False, color='g', linewidth=2)
-        axes[0].add_patch(rectangle_gt)
-        axes[0].add_patch(rectangle)
-
-    def draw_text_front(self, axes, uv, num):
-        axes[0].text(uv[0] + self.radius, uv[1] * self.y_scale - self.radius, str(num),
-                     fontsize=self.FONTSIZE, color=self.TEXTCOLOR, weight='bold')
-
-    def draw_text_bird(self, axes, idx, num):
         """Plot the number in the bird eye view map"""
 
         std = self.stds_epi[idx] if self.stds_epi[idx] > 0 else self.stds_ale[idx]
@@ -228,48 +317,128 @@ class Printer:
         delta_x = std * math.cos(theta)
         delta_z = std * math.sin(theta)
 
-        axes[1].text(self.xx_pred[idx] + delta_x, self.zz_pred[idx] + delta_z,
+        axes[1].text(self.xx_pred[idx] + delta_x + 0.2, self.zz_pred[idx] + delta_z + 0/2, chr(num),
-                     str(num), fontsize=self.FONTSIZE_BV, color='darkorange')
+                     fontsize=self.attr['fontsize_bv'],
+                     color=self.attr[self.modes[idx]]['numcolor'])
 
-    def draw_circle(self, axes, uv, color):
+    def _draw_uncertainty(self, axes, idx):
 
-        circle = Circle((uv[0], uv[1] * self.y_scale), radius=self.radius, color=color, fill=True)
+        theta = math.atan2(self.zz_pred[idx], self.xx_pred[idx])
-        axes[0].add_patch(circle)
+        dic_std = {'ale': self.stds_ale[idx], 'epi': self.stds_epi[idx]}
+        dic_x, dic_y = {}, {}
 
-    def set_axes(self, ax, axis):
+        # Aleatoric and epistemic
+        for key, std in dic_std.items():
+            delta_x = std * math.cos(theta)
+            delta_z = std * math.sin(theta)
+            dic_x[key] = (self.xx_pred[idx] - delta_x, self.xx_pred[idx] + delta_x)
+            dic_y[key] = (self.zz_pred[idx] - delta_z, self.zz_pred[idx] + delta_z)
+
+        # MonoLoco
+        if not self.auxs:
+            axes[1].plot(dic_x['epi'],
+                         dic_y['epi'],
+                         color='coral',
+                         linewidth=round(self.attr['linewidth']/2),
+                         label="Epistemic Uncertainty")
+
+            axes[1].plot(dic_x['ale'],
+                         dic_y['ale'],
+                         color='deepskyblue',
+                         linewidth=self.attr['linewidth'],
+                         label="Aleatoric Uncertainty")
+
+            axes[1].plot(self.xx_pred[idx],
+                         self.zz_pred[idx],
+                         color='cornflowerblue',
+                         label="Prediction",
+                         markersize=self.attr['markersize'],
+                         marker='o')
+
+            if self.gt[idx]:
+                axes[1].plot(self.xx_gt[idx],
+                             self.zz_gt[idx],
+                             color='k',
+                             label="Ground-truth",
+                             markersize=8,
+                             marker='x')
+
+        # MonStereo(stereo case)
+        elif self.auxs[idx] > 0.5:
+            axes[1].plot(dic_x['ale'],
+                         dic_y['ale'],
+                         color='r',
+                         linewidth=self.attr['linewidth'],
+                         label="Prediction (mono)")
+
+            axes[1].plot(dic_x['ale'],
+                         dic_y['ale'],
+                         color='deepskyblue',
+                         linewidth=self.attr['linewidth'],
+                         label="Prediction (stereo+mono)")
+
+            if self.gt[idx]:
+                axes[1].plot(self.xx_gt[idx],
+                             self.zz_gt[idx],
+                             color='k',
+                             label="Ground-truth",
+                             markersize=self.attr['markersize'],
+                             marker='x')
+
+        # MonStereo (monocular case)
+        else:
+            axes[1].plot(dic_x['ale'],
+                         dic_y['ale'],
+                         color='deepskyblue',
+                         linewidth=self.attr['linewidth'],
+                         label="Prediction (stereo+mono)")
+
+            axes[1].plot(dic_x['ale'],
+                         dic_y['ale'],
+                         color='r',
+                         linewidth=self.attr['linewidth'],
+                         label="Prediction (mono)")
+            if self.gt[idx]:
+                axes[1].plot(self.xx_gt[idx],
+                             self.zz_gt[idx],
+                             color='k',
+                             label="Ground-truth",
+                             markersize=self.attr['markersize'],
+                             marker='x')
+
+    def _draw_legend(self, axes):
+        # Bird eye view legend
+        if any(xx in self.output_types for xx in ['bird', 'multi']):
+            handles, labels = axes[1].get_legend_handles_labels()
+            by_label = OrderedDict(zip(labels, handles))
+            axes[1].legend(by_label.values(), by_label.keys(), loc='best', prop={'size': 15})
+
+    def _set_axes(self, ax, axis):
         assert axis in (0, 1)
 
         if axis == 0:
             ax.set_axis_off()
             ax.set_xlim(0, self.width)
             ax.set_ylim(self.height, 0)
+            if not self.activities or 'social_distance' not in self.activities:
                 self.mpl_im0 = ax.imshow(self.im)
             ax.get_xaxis().set_visible(False)
             ax.get_yaxis().set_visible(False)
 
         else:
+            line_style = 'w--' if self.webcam else 'k--'
             uv_max = [0., float(self.height)]
             xyz_max = pixel_to_camera(uv_max, self.kk, self.z_max)
             x_max = abs(xyz_max[0]) # shortcut to avoid oval circles in case of different kk
-            ax.plot([0, x_max], [0, self.z_max], 'k--')
+            corr = round(float(x_max / 3))
-            ax.plot([0, -x_max], [0, self.z_max], 'k--')
+            ax.plot([0, x_max], [0, self.z_max], line_style)
-            ax.set_ylim(0, self.z_max+1)
+            ax.plot([0, -x_max], [0, self.z_max], line_style)
+            ax.set_xlim(-x_max + corr, x_max - corr)
+            ax.set_ylim(0, self.z_max + 1)
             ax.set_xlabel("X [m]")
-            ax.set_ylabel("Z [m]")
+            if self.webcam:
-
+                ax.set_box_aspect(.8)
+                plt.xlim((-x_max, x_max))
+            plt.xticks(fontsize=self.attr['fontsize_ax'])
+            plt.yticks(fontsize=self.attr['fontsize_ax'])
         return ax
-
-
-def draw_legend(axes):
-    handles, labels = axes[1].get_legend_handles_labels()
-    by_label = OrderedDict(zip(labels, handles))
-    axes[1].legend(by_label.values(), by_label.keys())
-
-
-def get_angle(xx, zz):
-    """Obtain the points to plot the confidence of each annotation"""
-
-    theta = math.atan2(zz, xx)
-    angle = theta * (180 / math.pi)
-
-    return angle

monoloco/visuals/webcam.py

@@ -7,107 +7,181 @@ Implementation adapted from https://github.com/vita-epfl/openpifpaf/blob/master/
 """
 
 import time
+import logging
 
 import torch
 import matplotlib.pyplot as plt
 from PIL import Image
-import cv2
+try:
+    import cv2
+except ImportError:
+    cv2 = None
+
+import openpifpaf
+from openpifpaf import decoder, network, visualizer, show, logger
+import openpifpaf.datasets as datasets
 
 from ..visuals import Printer
-from ..network import PifPaf, MonoLoco
+from ..network import Loco
-from ..network.process import preprocess_pifpaf, factory_for_gt, image_transform
+from ..network.process import preprocess_pifpaf, factory_for_gt
+from ..predict import download_checkpoints
+
+LOG = logging.getLogger(__name__)
+
+def factory_from_args(args):
+
+    # Model
+    dic_models = download_checkpoints(args)
+    args.checkpoint = dic_models['keypoints']
+
+    logger.configure(args, LOG)  # logger first
+
+    assert len(args.output_types) == 1 and 'json' not in args.output_types
+
+    # Devices
+    args.device = torch.device('cpu')
+    args.pin_memory = False
+    if torch.cuda.is_available():
+        args.device = torch.device('cuda')
+        args.pin_memory = True
+    LOG.debug('neural network device: %s', args.device)
+
+    # Add visualization defaults
+    args.figure_width = 10
+    args.dpi_factor = 1.0
+
+    args.z_max = 10
+    args.show_all = True
+    args.no_save = True
+    args.batch_size = 1
+
+    if args.long_edge is None:
+        args.long_edge = 144
+    # Make default pifpaf argument
+    args.force_complete_pose = True
+    LOG.info("Force complete pose is active")
+
+    # Configure
+    decoder.configure(args)
+    network.Factory.configure(args)
+    show.configure(args)
+    visualizer.configure(args)
+
+    return args, dic_models
 
 
 def webcam(args):
 
-    # add args.device
+    assert args.mode in 'mono'
-    args.device = torch.device('cpu')
+    assert cv2
-    if torch.cuda.is_available():
-        args.device = torch.device('cuda')
 
-    # load models
+    args, dic_models = factory_from_args(args)
-    args.camera = True
+
-    pifpaf = PifPaf(args)
+    # Load Models
-    monoloco = MonoLoco(model=args.model, device=args.device)
+    net = Loco(model=dic_models[args.mode], mode=args.mode, device=args.device,
+               n_dropout=args.n_dropout, p_dropout=args.dropout)
+
+    # for openpifpaf predicitons
+    predictor = openpifpaf.Predictor(checkpoint=args.checkpoint)
 
     # Start recording
-    cam = cv2.VideoCapture(0)
+    cam = cv2.VideoCapture(args.camera)
-    visualizer_monoloco = None
+    visualizer_mono = None
 
     while True:
         start = time.time()
         ret, frame = cam.read()
-        image = cv2.resize(frame, None, fx=args.scale, fy=args.scale)
+        scale = (args.long_edge)/frame.shape[0]
+        image = cv2.resize(frame, None, fx=scale, fy=scale)
         height, width, _ = image.shape
-        print('resized image size: {}'.format(image.shape))
+        LOG.debug('resized image size: {}'.format(image.shape))
         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-        processed_image_cpu = image_transform(image.copy())
+        pil_image = Image.fromarray(image)
-        processed_image = processed_image_cpu.contiguous().to(args.device, non_blocking=True)
+
-        fields = pifpaf.fields(torch.unsqueeze(processed_image, 0))[0]
+        data = datasets.PilImageList(
-        _, _, pifpaf_out = pifpaf.forward(image, processed_image_cpu, fields)
+            [pil_image], preprocess=predictor.preprocess)
+
+        data_loader = torch.utils.data.DataLoader(
+            data, batch_size=1, shuffle=False,
+            pin_memory=False, collate_fn=datasets.collate_images_anns_meta)
+
+        for (_, _, _) in data_loader:
+
+            for idx, (preds, _, _) in enumerate(predictor.dataset(data)):
+
+                if idx == 0:
+                    pifpaf_outs = {
+                        'pred': preds,
+                        'left': [ann.json_data() for ann in preds],
+                        'image': image}
 
         if not ret:
             break
         key = cv2.waitKey(1)
-
         if key % 256 == 27:
             # ESC pressed
-            print("Escape hit, closing...")
+            LOG.info("Escape hit, closing...")
             break
-        pil_image = Image.fromarray(image)
-        intrinsic_size = [xx * 1.3 for xx in pil_image.size]
-        kk, dict_gt = factory_for_gt(intrinsic_size)  # better intrinsics for mac camera
-        if visualizer_monoloco is None:  # it is, at the beginning
-            visualizer_monoloco = VisualizerMonoloco(kk, args)(pil_image)  # create it with the first image
-            visualizer_monoloco.send(None)
 
-        boxes, keypoints = preprocess_pifpaf(pifpaf_out, (width, height))
+        kk, dic_gt = factory_for_gt(pil_image.size, focal_length=args.focal)
-        outputs, varss = monoloco.forward(keypoints, kk)
+        boxes, keypoints = preprocess_pifpaf(
-        dic_out = monoloco.post_process(outputs, varss, boxes, keypoints, kk, dict_gt)
+            pifpaf_outs['left'], (width, height))
-        print(dic_out)
+
-        visualizer_monoloco.send((pil_image, dic_out))
+        dic_out = net.forward(keypoints, kk)
+        dic_out = net.post_process(dic_out, boxes, keypoints, kk, dic_gt)
+
+        if 'social_distance' in args.activities:
+            dic_out = net.social_distance(dic_out, args)
+        if 'raise_hand' in args.activities:
+            dic_out = net.raising_hand(dic_out, keypoints)
+        if visualizer_mono is None:  # it is, at the beginning
+            visualizer_mono = Visualizer(kk, args)(pil_image)  # create it with the first image
+            visualizer_mono.send(None)
+
+        LOG.debug(dic_out)
+        visualizer_mono.send((pil_image, dic_out, pifpaf_outs))
 
         end = time.time()
-        print("run-time: {:.2f} ms".format((end-start)*1000))
+        LOG.info("run-time: {:.2f} ms".format((end-start)*1000))
 
     cam.release()
 
     cv2.destroyAllWindows()
 
 
-class VisualizerMonoloco:
+class Visualizer:
-    def __init__(self, kk, args, epistemic=False):
+    def __init__(self, kk, args):
         self.kk = kk
         self.args = args
-        self.z_max = args.z_max
-        self.epistemic = epistemic
-        self.output_types = args.output_types
 
-    def __call__(self, first_image, fig_width=4.0, **kwargs):
+    def __call__(self, first_image, fig_width=1.0, **kwargs):
         if 'figsize' not in kwargs:
-            kwargs['figsize'] = (fig_width, fig_width * first_image.size[0] / first_image.size[1])
+            kwargs['figsize'] = (fig_width, fig_width *
+                                 first_image.size[0] / first_image.size[1])
 
-        printer = Printer(first_image, output_path="", kk=self.kk, output_types=self.output_types,
+        printer = Printer(first_image, output_path="",
-                          z_max=self.z_max, epistemic=self.epistemic)
+                          kk=self.kk, args=self.args)
-        figures, axes = printer.factory_axes()
+
+        figures, axes = printer.factory_axes(None)
 
         for fig in figures:
             fig.show()
 
         while True:
-            image, dict_ann = yield
+            image, dic_out, pifpaf_outs = yield
-            while axes and (axes[-1] and axes[-1].patches):  # for front -1==0, for bird/combined -1 == 1
+
-                if axes[0]:
+            # Clears previous annotations between frames
-                    del axes[0].patches[0]
+            axes[0].patches = []
-                    del axes[0].texts[0]
+            axes[0].lines = []
-                if len(axes) == 2:
+            axes[0].texts = []
-                    del axes[1].patches[0]
+            if len(axes) > 1:
-                    del axes[1].patches[0]  # the one became the 0
+                axes[1].patches = []
-                    if len(axes[1].lines) > 2:
+                axes[1].lines = [axes[1].lines[0], axes[1].lines[1]]
-                        del axes[1].lines[2]
+                axes[1].texts = []
-                        if axes[1].texts:  # in case of no text
+
-                            del axes[1].texts[0]
+            if dic_out and dic_out['dds_pred']:
-            printer.draw(figures, axes, dict_ann, image)
+                printer._process_results(dic_out)
+                printer.draw(figures, axes, image, dic_out, pifpaf_outs['left'])
                 mypause(0.01)
 
 

pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["setuptools", "versioneer-518"]
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+testpaths = ["tests"]
+markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"]
+
+[tool.pylint.BASIC]
+class-const-naming-style = "snake_case"  # checked since pylint 2.7.3
+
+[tool.pylint.IMPORTS]
+allow-any-import-level = []
+
+[tool.pylint.SIMILARITIES]
+min-similarity-lines = 15  # Minimum lines number of a similarity.
+ignore-comments = "yes"  # Ignore comments when computing similarities.
+ignore-docstrings = "yes"  # Ignore docstrings when computing similarities.
+ignore-imports = "yes"  # Ignore imports when computing similarities.
+
+[tool.pylint.TYPECHECK]
+generated-members = ["numpy.*", "torch.*", "cv2.*", "openpifpaf.functional.*"]
+ignored-modules = ["openpifpaf.functional"]
+disable = [
+    "missing-docstring",
+    "too-many-arguments",
+    "too-many-instance-attributes",
+    "too-many-locals",
+    "too-few-public-methods",
+    "unsubscriptable-object",
+    "not-callable",  # for torch tensors
+    "invalid-name",
+    "logging-format-interpolation",
+    ]
+[tool.pylint.FORMAT]
+max-line-length = 120
+

setup.cfg

@@ -0,0 +1,13 @@
+
+[versioneer]
+VCS = git
+style = pep440
+versionfile_source = monoloco/_version.py
+versionfile_build = monoloco/_version.py
+tag_prefix = v
+#parentdir_prefix =
+
+[pycodestyle]
+max-line-length=120
+ignore=E731,E741,W503
+exclude=monoloco/__init__.py

setup.py

@@ -1,13 +1,18 @@
+
 from setuptools import setup
 
-# extract version from __init__.py
+# This is needed for versioneer to be importable when building with PEP 517.
-with open('monoloco/__init__.py', 'r') as f:
+# See <https://github.com/warner/python-versioneer/issues/193> and links
-    VERSION_LINE = [l for l in f if l.startswith('__version__')][0]
+# therein for more information.
-    VERSION = VERSION_LINE.split('=')[1].strip()[1:-1]
+
+import os, sys
+sys.path.append(os.path.dirname(__file__))
+import versioneer
 
 setup(
     name='monoloco',
-    version=VERSION,
+    version=versioneer.get_version(),
+    cmdclass=versioneer.get_cmdclass(),
     packages=[
         'monoloco',
         'monoloco.network',
@@ -18,7 +23,7 @@ setup(
         'monoloco.utils'
     ],
     license='GNU AGPLv3',
-    description='MonoLoco: Monocular 3D Pedestrian Localization and Uncertainty Estimation',
+    description=' A 3D vision library from 2D keypoints',
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
     author='Lorenzo Bertoni',
@@ -27,19 +32,23 @@ setup(
     zip_safe=False,
 
     install_requires=[
-        'torch<=1.1.0',
+        'openpifpaf>=v0.12.10',
-        'Pillow<=6.3',
+        'matplotlib',
-        'torchvision<=0.3.0',
-        'openpifpaf<=0.9.0',
-        'tabulate<=0.8.3',   # For evaluation
     ],
     extras_require={
         'test': [
-            'pylint<=2.4.2',
+            'pylint',
-            'pytest<=4.6.3',
+            'pytest',
+            'gdown',
+            'scipy',  # for social distancing gaussian blur
+        ],
+        'eval': [
+            'tabulate',
+            'sklearn',
+            'pandas',
         ],
         'prep': [
-            'nuscenes-devkit<=1.0.2',
+            'nuscenes-devkit==1.0.2',
         ],
     },
 )

tests/test_package.py

@@ -1,69 +0,0 @@
-"""Test if the main modules of the package run correctly"""
-
-import os
-import sys
-import json
-
-# Python does not consider the current directory to be a package
-sys.path.insert(0, os.path.join('..', 'monoloco'))
-
-from PIL import Image
-
-from monoloco.train import Trainer
-from monoloco.network import MonoLoco
-from monoloco.network.process import preprocess_pifpaf, factory_for_gt
-from monoloco.visuals.printer import Printer
-
-JOINTS = 'tests/joints_sample.json'
-PIFPAF_KEYPOINTS = 'tests/002282.png.pifpaf.json'
-IMAGE = 'docs/002282.png'
-
-
-def tst_trainer(joints):
-    trainer = Trainer(joints=joints, epochs=150, lr=0.01)
-    _ = trainer.train()
-    dic_err, model = trainer.evaluate()
-    return dic_err['val']['all']['mean'], model
-
-
-def tst_prediction(model, path_keypoints):
-    with open(path_keypoints, 'r') as f:
-        pifpaf_out = json.load(f)
-
-    kk, _ = factory_for_gt(im_size=[1240, 340])
-
-    # Preprocess pifpaf outputs and run monoloco
-    boxes, keypoints = preprocess_pifpaf(pifpaf_out)
-    monoloco = MonoLoco(model)
-    outputs, varss = monoloco.forward(keypoints, kk)
-    dic_out = monoloco.post_process(outputs, varss, boxes, keypoints, kk)
-    return dic_out, kk
-
-
-def tst_printer(dic_out, kk, image_path):
-    """Draw a fake figure"""
-    with open(image_path, 'rb') as f:
-        pil_image = Image.open(f).convert('RGB')
-    printer = Printer(image=pil_image, output_path='tests/test_image', kk=kk, output_types=['combined'], z_max=15)
-    figures, axes = printer.factory_axes()
-    printer.draw(figures, axes, dic_out, pil_image, save=True)
-
-
-def test_package():
-
-    # Training test
-    val_acc, model = tst_trainer(JOINTS)
-    assert val_acc < 2.5
-
-    # Prediction test
-    dic_out, kk = tst_prediction(model, PIFPAF_KEYPOINTS)
-    assert dic_out['boxes'] and kk
-
-    # Visualization test
-    tst_printer(dic_out, kk, IMAGE)
-
-
-
-
-
-

tests/test_train_mono.py

@@ -0,0 +1,79 @@
+
+"""
+Adapted from https://github.com/openpifpaf/openpifpaf/blob/main/tests/test_train.py,
+which is: 'Copyright 2019-2021 by Sven Kreiss and contributors. All rights reserved.'
+and licensed under GNU AGPLv3
+"""
+
+import os
+import subprocess
+
+import gdown
+
+OPENPIFPAF_MODEL = 'https://drive.google.com/uc?id=1b408ockhh29OLAED8Tysd2yGZOo0N_SQ'
+
+TRAIN_COMMAND = [
+    'python3', '-m', 'monoloco.run',
+    'train',
+    '--joints', 'tests/sample_joints-kitti-mono.json',
+    '--lr=0.001',
+    '-e=10',
+    ]
+
+
+PREDICT_COMMAND = [
+    'python3', '-m', 'monoloco.run',
+    'predict',
+    'docs/002282.png',
+    '--output_types', 'multi', 'json',
+    '--decoder-workers=0'  # for windows
+]
+
+PREDICT_COMMAND_SOCIAL_DISTANCE = [
+    'python3', '-m', 'monoloco.run',
+    'predict',
+    'docs/frame0032.jpg',
+    '--activities', 'social_distance',
+    '--output_types', 'front', 'bird',
+    '--decoder-workers=0'  # for windows'
+]
+
+
+def test_train_mono(tmp_path):
+    # train a model
+    train_cmd = TRAIN_COMMAND + ['--out={}'.format(os.path.join(tmp_path, 'train_test.pkl'))]
+    print(' '.join(train_cmd))
+    subprocess.run(train_cmd, check=True, capture_output=True)
+    print(os.listdir(tmp_path))
+
+    # find the trained model checkpoint and download pifpaf one
+    final_model = next(iter(f for f in os.listdir(tmp_path) if f.endswith('.pkl')))
+    pifpaf_model = os.path.join(tmp_path, 'pifpaf_model.pkl')
+    print('Downloading OpenPifPaf model in temporary folder')
+    gdown.download(OPENPIFPAF_MODEL, pifpaf_model)
+
+    # run predictions with that model
+    model = os.path.join(tmp_path, final_model)
+
+    print(model)
+    predict_cmd = PREDICT_COMMAND + [
+        '--model={}'.format(model),
+        '--checkpoint={}'.format(pifpaf_model),
+        '-o={}'.format(tmp_path),
+    ]
+    print(' '.join(predict_cmd))
+    subprocess.run(predict_cmd, check=True, capture_output=True)
+    print(os.listdir(tmp_path))
+    assert 'out_002282.png.multi.png' in os.listdir(tmp_path)
+    assert 'out_002282.png.monoloco.json' in os.listdir(tmp_path)
+
+    predict_cmd_sd = PREDICT_COMMAND_SOCIAL_DISTANCE + [
+        '--model={}'.format(model),
+        '--checkpoint={}'.format(pifpaf_model),
+        '-o={}'.format(tmp_path),
+    ]
+    print(' '.join(predict_cmd_sd))
+    subprocess.run(predict_cmd_sd, check=True, capture_output=True)
+    print(os.listdir(tmp_path))
+    assert 'out_frame0032.jpg.front.png' in os.listdir(tmp_path)
+    assert 'out_frame0032.jpg.bird.png' in os.listdir(tmp_path)

tests/test_train_stereo.py

@@ -0,0 +1,59 @@
+
+"""
+Adapted from https://github.com/openpifpaf/openpifpaf/blob/main/tests/test_train.py,
+which is: 'Copyright 2019-2021 by Sven Kreiss and contributors. All rights reserved.'
+and licensed under GNU AGPLv3
+"""
+
+import os
+import subprocess
+
+import gdown
+
+OPENPIFPAF_MODEL = 'https://drive.google.com/uc?id=1b408ockhh29OLAED8Tysd2yGZOo0N_SQ'
+
+TRAIN_COMMAND = [
+    'python3', '-m', 'monoloco.run',
+    'train',
+    '--mode=stereo',
+    '--joints', 'tests/sample_joints-kitti-stereo.json',
+    '--lr=0.001',
+    '-e=20',
+    ]
+
+
+PREDICT_COMMAND = [
+    'python3', '-m', 'monoloco.run',
+    'predict',
+    '--mode=stereo',
+    '--glob', 'docs/000840*.png',
+    '--output_types', 'multi', 'json',
+    '--decoder-workers=0',  # for windows'
+]
+
+
+def test_train_stereo(tmp_path):
+    # train a model
+    train_cmd = TRAIN_COMMAND + ['--out={}'.format(os.path.join(tmp_path, 'train_test.pkl'))]
+    print(' '.join(train_cmd))
+    subprocess.run(train_cmd, check=True, capture_output=True)
+    print(os.listdir(tmp_path))
+
+    # find the trained model checkpoint
+    final_model = next(iter(f for f in os.listdir(tmp_path) if f.endswith('.pkl')))
+    pifpaf_model = os.path.join(tmp_path, 'pifpaf_model.pkl')
+    print('Downloading OpenPifPaf model in temporary folder')
+    gdown.download(OPENPIFPAF_MODEL, pifpaf_model)
+
+    # run predictions with that model
+    model = os.path.join(tmp_path, final_model)
+
+    predict_cmd = PREDICT_COMMAND + [
+        '--model={}'.format(model),
+        '--checkpoint={}'.format(pifpaf_model),
+        '-o={}'.format(tmp_path),
+    ]
+    print(' '.join(predict_cmd))
+    subprocess.run(predict_cmd, check=True, capture_output=True)
+    print(os.listdir(tmp_path))
+    assert 'out_000840.png.multi.png' in os.listdir(tmp_path)

tests/test_utils.py

@@ -1,12 +1,14 @@
 import os
 import sys
 
+from monoloco.utils import pixel_to_camera
+from monoloco.utils import get_iou_matrix
+
 # Python does not consider the current directory to be a package
 sys.path.insert(0, os.path.join('..', 'monoloco'))
 
 
 def test_iou():
-    from monoloco.utils import get_iou_matrix
     boxes_pred = [[1, 100, 1, 200]]
     boxes_gt = [[100., 120., 150., 160.],[12, 110, 130., 160.]]
     iou_matrix = get_iou_matrix(boxes_pred, boxes_gt)
@@ -14,7 +16,6 @@ def test_iou():
 
 
 def test_pixel_to_camera():
-    from monoloco.utils import pixel_to_camera
     kk = [[718.3351, 0., 600.3891], [0., 718.3351, 181.5122], [0., 0., 1.]]
     zz = 10
     uv_vector = [1000., 400.]

tests/test_visuals.py

@@ -1,23 +0,0 @@
-import os
-import sys
-from collections import defaultdict
-
-from PIL import Image
-
-# Python does not consider the current directory to be a package
-sys.path.insert(0, os.path.join('..', 'monoloco'))
-
-
-def test_printer():
-    """Draw a fake figure"""
-    from monoloco.visuals.printer import Printer
-    test_list = [[718.3351, 0., 600.3891], [0., 718.3351, 181.5122], [0., 0., 1.]]
-    boxes = [xx + [0] for xx in test_list]
-    kk = test_list
-    dict_ann = defaultdict(lambda: [1., 2., 3.], xyz_real=test_list, xyz_pred=test_list, uv_shoulders=test_list,
-                           boxes=boxes, boxes_gt=boxes)
-    with open('docs/002282.png', 'rb') as f:
-        pil_image = Image.open(f).convert('RGB')
-    printer = Printer(image=pil_image, output_path=None, kk=kk, output_types=['combined'])
-    figures, axes = printer.factory_axes()
-    printer.draw(figures, axes, dict_ann, pil_image)