├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── apa-6th-edition.csl ├── chapter-01.md ├── chapter-02.md ├── chapter-03.md ├── chapter-04.md ├── chapter-05.md ├── chapter-06.md ├── chapter-07.md ├── chapter-08.md ├── chapter-09.md ├── chapter-10.md ├── cover.png ├── cover.svg ├── endmatter.md ├── figures ├── fig_Fuster01_2.png ├── fig_SommerWurtz00_fig2.png ├── fig_ab_ac_list.png ├── fig_actor_critic_basic.png ├── fig_adex_vm_vs_hz.png ├── fig_aphasia_areas.png ├── fig_attractor.png ├── fig_attractor_cal.png ├── fig_bcm_function.png ├── fig_bg_action_sel_dam.png ├── fig_bg_frontal_da_burst_dip.png ├── fig_bg_gating_science.png ├── fig_bg_loops_ads86.png ├── fig_bg_opal_hilow.png ├── fig_bidir_backprop_intuition.png ├── fig_bp_compute_delta.png ├── fig_brain_anatomy.png ├── fig_brain_and_lobes.png ├── fig_brain_and_lobes.svg ├── fig_brain_puzzle_behav_bio_data.png ├── fig_brain_puzzle_blue_sky_pieces.png ├── fig_brodmann_areas.png ├── fig_brodmann_areas_3d_color.png ├── fig_brodmann_areas_color.png ├── fig_bvpvlv_functional_org_simpler.png ├── fig_bvpvlv_sim1a.png ├── fig_category_hierarch_dist_reps.png ├── fig_ccnbook_cover.png ├── fig_cerebellum.png ├── fig_cerebellum_cortex_inputs.png ├── fig_coarse_coding.png ├── fig_cortex_bidir_cons_map.png ├── fig_cortex_bio_arealayers.png ├── fig_cortex_bio_layers.jpg ├── fig_cortex_bio_neurtypes.png ├── fig_cortex_lobes.png ├── fig_cortical_cons_ff_fb_lat.png ├── fig_cortical_fun_org_tins_acc_ofc.png ├── fig_cortical_layers_in_hid_out.png ├── fig_cortical_lobes.jpg ├── fig_cortical_neuron.png ├── fig_dalmatian.png ├── fig_diffusion.png ├── fig_dist_rep_vis_bio.png ├── fig_distrib_sem.png ├── fig_dyslex_sem_clust.png ├── fig_electricity.png ├── fig_expect_outcome_errs.png ├── fig_face_categ_dim_prjn.png ├── fig_felleman_vanessen.png ├── fig_frank_badre12_pfc_hierarch.png ├── fig_gears.png ├── fig_generec_compute_delta.png ├── fig_halle_berry_neuron.jpg ├── fig_hasselmo_bodelon_wyble_theta.png ├── fig_haxbyetal01_obj_maps.jpg ├── fig_hcmp_rfs.png ├── fig_hebb_demo_blank.png ├── fig_hikosaka_gating.png ├── fig_hippo_comp_learn_sys.png ├── fig_hippo_mem_formation.png ├── fig_inhib_types.png ├── fig_integ_lrate.png ├── fig_invar_trans.png ├── fig_ions.png ├── fig_ipa_chart_consonants_simple.png ├── fig_ipa_chart_vowels_ipa_pmsp.png ├── fig_kirkwood_et_al_96_bcm_thresh.png ├── fig_kobatake_tanaka_94_invariance.png ├── fig_kobatake_tanaka_94_v2_v4_it.png ├── fig_lang_paths.png ├── fig_lang_phrase_stru.png ├── fig_leabra_mechs_xcal.png ├── fig_leutgeb_et_al_07_dg_patsep.png ├── fig_location_terms.png ├── fig_lookup_table_function_approximator.png ├── fig_ltp_ltd_ca2+.png ├── fig_ltpd_synapse.png ├── fig_markov_et_al_cort_hier.png ├── fig_mnt2out.png ├── fig_neglect_drawings.png ├── fig_neglect_line_bisect.png ├── fig_neglect_portrait.jpg ├── fig_neuron_as_detect.png ├── fig_neuron_rate_code_approx.png ├── fig_neuron_spiking.png ├── fig_nxx1_fun.png ├── fig_objrec_difficulty.png ├── fig_objrec_objs.png ├── fig_on_off_rfields_edge.png ├── fig_patsep_clr.png ├── fig_pbwm_architecture_bio.png ├── fig_pbwm_architecture_mnt_out.png ├── fig_pfc_trc_reverb_loops.png ├── fig_posner_disengage_mech.png ├── fig_posner_graph.png ├── fig_posner_task.png ├── fig_pvlv_bio_no_cereb.png ├── fig_pvlv_da.png ├── fig_reading_model.png ├── fig_schultz97_vta_td.png ├── fig_sg_net_deepleabra.png ├── fig_sg_noun_clust_deep.png ├── fig_sg_sent_probe_clust_deep.png ├── fig_spat_attn_lateral.png ├── fig_stdp_bipoo98.png ├── fig_stroop.png ├── fig_stroop_data.png ├── fig_synapse.png ├── fig_synapse_em.jpg ├── fig_tanaka03_topo.png ├── fig_tanaka03_topo_maps.png ├── fig_urakubo_et_al_model.png ├── fig_v1_orientation_cols_data.jpg ├── fig_v1_orientation_tuning_data.jpg ├── fig_v1_simple_dog_edge.png ├── fig_v1_visual_filters_hypercols_basic.png ├── fig_v1rf_map.png ├── fig_vert_line_detector_probs.png ├── fig_vis_optic_path.png ├── fig_vis_system_bio.png ├── fig_vm_as_tug_of_war.png ├── fig_vm_as_tug_of_war_cases.png ├── fig_vocal_tract.png ├── fig_wheres_waldo.jpg ├── fig_wt_contrast_sigmoid_fun.png ├── fig_xcal_bcm_err_learn.png ├── fig_xcal_bcm_selforg_learn.png ├── fig_xcal_dwt_fun.png ├── fig_xcal_dwt_fun_urakubo_fit_full.png └── web_header.png ├── frontmatter.md ├── glossary.md ├── metadata.yaml └── references.bib /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | /.idea 3 | /tmp 4 | book.epub 5 | book.json 6 | book.html 7 | book.docx 8 | book.pdf 9 | book.latex 10 | doit-db.json 11 | __pycache__ 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Attribution 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution 4.0 International Public License 58 | 59 | By exercising the Licensed Rights (defined below), You accept and agree 60 | to be bound by the terms and conditions of this Creative Commons 61 | Attribution 4.0 International Public License ("Public License"). To the 62 | extent this Public License may be interpreted as a contract, You are 63 | granted the Licensed Rights in consideration of Your acceptance of 64 | these terms and conditions, and the Licensor grants You such rights in 65 | consideration of benefits the Licensor receives from making the 66 | Licensed Material available under these terms and conditions. 67 | 68 | 69 | Section 1 -- Definitions. 70 | 71 | a. Adapted Material means material subject to Copyright and Similar 72 | Rights that is derived from or based upon the Licensed Material 73 | and in which the Licensed Material is translated, altered, 74 | arranged, transformed, or otherwise modified in a manner requiring 75 | permission under the Copyright and Similar Rights held by the 76 | Licensor. For purposes of this Public License, where the Licensed 77 | Material is a musical work, performance, or sound recording, 78 | Adapted Material is always produced where the Licensed Material is 79 | synched in timed relation with a moving image. 80 | 81 | b. Adapter's License means the license You apply to Your Copyright 82 | and Similar Rights in Your contributions to Adapted Material in 83 | accordance with the terms and conditions of this Public License. 84 | 85 | c. Copyright and Similar Rights means copyright and/or similar rights 86 | closely related to copyright including, without limitation, 87 | performance, broadcast, sound recording, and Sui Generis Database 88 | Rights, without regard to how the rights are labeled or 89 | categorized. For purposes of this Public License, the rights 90 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 91 | Rights. 92 | 93 | d. Effective Technological Measures means those measures that, in the 94 | absence of proper authority, may not be circumvented under laws 95 | fulfilling obligations under Article 11 of the WIPO Copyright 96 | Treaty adopted on December 20, 1996, and/or similar international 97 | agreements. 98 | 99 | e. Exceptions and Limitations means fair use, fair dealing, and/or 100 | any other exception or limitation to Copyright and Similar Rights 101 | that applies to Your use of the Licensed Material. 102 | 103 | f. Licensed Material means the artistic or literary work, database, 104 | or other material to which the Licensor applied this Public 105 | License. 106 | 107 | g. Licensed Rights means the rights granted to You subject to the 108 | terms and conditions of this Public License, which are limited to 109 | all Copyright and Similar Rights that apply to Your use of the 110 | Licensed Material and that the Licensor has authority to license. 111 | 112 | h. Licensor means the individual(s) or entity(ies) granting rights 113 | under this Public License. 114 | 115 | i. Share means to provide material to the public by any means or 116 | process that requires permission under the Licensed Rights, such 117 | as reproduction, public display, public performance, distribution, 118 | dissemination, communication, or importation, and to make material 119 | available to the public including in ways that members of the 120 | public may access the material from a place and at a time 121 | individually chosen by them. 122 | 123 | j. Sui Generis Database Rights means rights other than copyright 124 | resulting from Directive 96/9/EC of the European Parliament and of 125 | the Council of 11 March 1996 on the legal protection of databases, 126 | as amended and/or succeeded, as well as other essentially 127 | equivalent rights anywhere in the world. 128 | 129 | k. You means the individual or entity exercising the Licensed Rights 130 | under this Public License. Your has a corresponding meaning. 131 | 132 | 133 | Section 2 -- Scope. 134 | 135 | a. License grant. 136 | 137 | 1. Subject to the terms and conditions of this Public License, 138 | the Licensor hereby grants You a worldwide, royalty-free, 139 | non-sublicensable, non-exclusive, irrevocable license to 140 | exercise the Licensed Rights in the Licensed Material to: 141 | 142 | a. reproduce and Share the Licensed Material, in whole or 143 | in part; and 144 | 145 | b. produce, reproduce, and Share Adapted Material. 146 | 147 | 2. Exceptions and Limitations. For the avoidance of doubt, where 148 | Exceptions and Limitations apply to Your use, this Public 149 | License does not apply, and You do not need to comply with 150 | its terms and conditions. 151 | 152 | 3. Term. The term of this Public License is specified in Section 153 | 6(a). 154 | 155 | 4. Media and formats; technical modifications allowed. The 156 | Licensor authorizes You to exercise the Licensed Rights in 157 | all media and formats whether now known or hereafter created, 158 | and to make technical modifications necessary to do so. The 159 | Licensor waives and/or agrees not to assert any right or 160 | authority to forbid You from making technical modifications 161 | necessary to exercise the Licensed Rights, including 162 | technical modifications necessary to circumvent Effective 163 | Technological Measures. For purposes of this Public License, 164 | simply making modifications authorized by this Section 2(a) 165 | (4) never produces Adapted Material. 166 | 167 | 5. Downstream recipients. 168 | 169 | a. Offer from the Licensor -- Licensed Material. Every 170 | recipient of the Licensed Material automatically 171 | receives an offer from the Licensor to exercise the 172 | Licensed Rights under the terms and conditions of this 173 | Public License. 174 | 175 | b. No downstream restrictions. You may not offer or impose 176 | any additional or different terms or conditions on, or 177 | apply any Effective Technological Measures to, the 178 | Licensed Material if doing so restricts exercise of the 179 | Licensed Rights by any recipient of the Licensed 180 | Material. 181 | 182 | 6. No endorsement. Nothing in this Public License constitutes or 183 | may be construed as permission to assert or imply that You 184 | are, or that Your use of the Licensed Material is, connected 185 | with, or sponsored, endorsed, or granted official status by, 186 | the Licensor or others designated to receive attribution as 187 | provided in Section 3(a)(1)(A)(i). 188 | 189 | b. Other rights. 190 | 191 | 1. Moral rights, such as the right of integrity, are not 192 | licensed under this Public License, nor are publicity, 193 | privacy, and/or other similar personality rights; however, to 194 | the extent possible, the Licensor waives and/or agrees not to 195 | assert any such rights held by the Licensor to the limited 196 | extent necessary to allow You to exercise the Licensed 197 | Rights, but not otherwise. 198 | 199 | 2. Patent and trademark rights are not licensed under this 200 | Public License. 201 | 202 | 3. To the extent possible, the Licensor waives any right to 203 | collect royalties from You for the exercise of the Licensed 204 | Rights, whether directly or through a collecting society 205 | under any voluntary or waivable statutory or compulsory 206 | licensing scheme. In all other cases the Licensor expressly 207 | reserves any right to collect such royalties. 208 | 209 | 210 | Section 3 -- License Conditions. 211 | 212 | Your exercise of the Licensed Rights is expressly made subject to the 213 | following conditions. 214 | 215 | a. Attribution. 216 | 217 | 1. If You Share the Licensed Material (including in modified 218 | form), You must: 219 | 220 | a. retain the following if it is supplied by the Licensor 221 | with the Licensed Material: 222 | 223 | i. identification of the creator(s) of the Licensed 224 | Material and any others designated to receive 225 | attribution, in any reasonable manner requested by 226 | the Licensor (including by pseudonym if 227 | designated); 228 | 229 | ii. a copyright notice; 230 | 231 | iii. a notice that refers to this Public License; 232 | 233 | iv. a notice that refers to the disclaimer of 234 | warranties; 235 | 236 | v. a URI or hyperlink to the Licensed Material to the 237 | extent reasonably practicable; 238 | 239 | b. indicate if You modified the Licensed Material and 240 | retain an indication of any previous modifications; and 241 | 242 | c. indicate the Licensed Material is licensed under this 243 | Public License, and include the text of, or the URI or 244 | hyperlink to, this Public License. 245 | 246 | 2. You may satisfy the conditions in Section 3(a)(1) in any 247 | reasonable manner based on the medium, means, and context in 248 | which You Share the Licensed Material. For example, it may be 249 | reasonable to satisfy the conditions by providing a URI or 250 | hyperlink to a resource that includes the required 251 | information. 252 | 253 | 3. If requested by the Licensor, You must remove any of the 254 | information required by Section 3(a)(1)(A) to the extent 255 | reasonably practicable. 256 | 257 | 4. If You Share Adapted Material You produce, the Adapter's 258 | License You apply must not prevent recipients of the Adapted 259 | Material from complying with this Public License. 260 | 261 | 262 | Section 4 -- Sui Generis Database Rights. 263 | 264 | Where the Licensed Rights include Sui Generis Database Rights that 265 | apply to Your use of the Licensed Material: 266 | 267 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 268 | to extract, reuse, reproduce, and Share all or a substantial 269 | portion of the contents of the database; 270 | 271 | b. if You include all or a substantial portion of the database 272 | contents in a database in which You have Sui Generis Database 273 | Rights, then the database in which You have Sui Generis Database 274 | Rights (but not its individual contents) is Adapted Material; and 275 | 276 | c. You must comply with the conditions in Section 3(a) if You Share 277 | all or a substantial portion of the contents of the database. 278 | 279 | For the avoidance of doubt, this Section 4 supplements and does not 280 | replace Your obligations under this Public License where the Licensed 281 | Rights include other Copyright and Similar Rights. 282 | 283 | 284 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 285 | 286 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 287 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 288 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 289 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 290 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 291 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 292 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 293 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 294 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 295 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 296 | 297 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 298 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 299 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 300 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 301 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 302 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 303 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 304 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 305 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 306 | 307 | c. The disclaimer of warranties and limitation of liability provided 308 | above shall be interpreted in a manner that, to the extent 309 | possible, most closely approximates an absolute disclaimer and 310 | waiver of all liability. 311 | 312 | 313 | Section 6 -- Term and Termination. 314 | 315 | a. This Public License applies for the term of the Copyright and 316 | Similar Rights licensed here. However, if You fail to comply with 317 | this Public License, then Your rights under this Public License 318 | terminate automatically. 319 | 320 | b. Where Your right to use the Licensed Material has terminated under 321 | Section 6(a), it reinstates: 322 | 323 | 1. automatically as of the date the violation is cured, provided 324 | it is cured within 30 days of Your discovery of the 325 | violation; or 326 | 327 | 2. upon express reinstatement by the Licensor. 328 | 329 | For the avoidance of doubt, this Section 6(b) does not affect any 330 | right the Licensor may have to seek remedies for Your violations 331 | of this Public License. 332 | 333 | c. For the avoidance of doubt, the Licensor may also offer the 334 | Licensed Material under separate terms or conditions or stop 335 | distributing the Licensed Material at any time; however, doing so 336 | will not terminate this Public License. 337 | 338 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 339 | License. 340 | 341 | 342 | Section 7 -- Other Terms and Conditions. 343 | 344 | a. The Licensor shall not be bound by any additional or different 345 | terms or conditions communicated by You unless expressly agreed. 346 | 347 | b. Any arrangements, understandings, or agreements regarding the 348 | Licensed Material not stated herein are separate from and 349 | independent of the terms and conditions of this Public License. 350 | 351 | 352 | Section 8 -- Interpretation. 353 | 354 | a. For the avoidance of doubt, this Public License does not, and 355 | shall not be interpreted to, reduce, limit, restrict, or impose 356 | conditions on any use of the Licensed Material that could lawfully 357 | be made without permission under this Public License. 358 | 359 | b. To the extent possible, if any provision of this Public License is 360 | deemed unenforceable, it shall be automatically reformed to the 361 | minimum extent necessary to make it enforceable. If the provision 362 | cannot be reformed, it shall be severed from this Public License 363 | without affecting the enforceability of the remaining terms and 364 | conditions. 365 | 366 | c. No term or condition of this Public License will be waived and no 367 | failure to comply consented to unless expressly agreed to by the 368 | Licensor. 369 | 370 | d. Nothing in this Public License constitutes or may be interpreted 371 | as a limitation upon, or waiver of, any privileges and immunities 372 | that apply to the Licensor or You, including from the legal 373 | processes of any jurisdiction or authority. 374 | 375 | 376 | ======================================================================= 377 | 378 | Creative Commons is not a party to its public 379 | licenses. Notwithstanding, Creative Commons may elect to apply one of 380 | its public licenses to material it publishes and in those instances 381 | will be considered the “Licensor.” The text of the Creative Commons 382 | public licenses is dedicated to the public domain under the CC0 Public 383 | Domain Dedication. Except for the limited purpose of indicating that 384 | material is shared under a Creative Commons public license or as 385 | otherwise permitted by the Creative Commons policies published at 386 | creativecommons.org/policies, Creative Commons does not authorize the 387 | use of the trademark "Creative Commons" or any other trademark or logo 388 | of Creative Commons without its prior written consent including, 389 | without limitation, in connection with any unauthorized modifications 390 | to any of its public licenses or any other arrangements, 391 | understandings, or agreements concerning use of licensed material. For 392 | the avoidance of doubt, this paragraph does not form part of the 393 | public licenses. 394 | 395 | Creative Commons may be contacted at creativecommons.org. 396 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # makefile for building book 2 | 3 | all: 4 | author book -o ccnbook_ed5 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Computational Cognitive Neuroscience, Fifth Edition 2 | 3 | This is the 5th edition of the online, freely available textbook, providing a complete, self-contained introduction to the field of **Computational Cognitive Neuroscience**, where computer models of the brain are used to understand a wide range of cognitive functions, including perception, attention, learning, memory, motor control, executive function, and language. 4 | 5 | The first part of this textbook develops a coherent set of computational and neural principles that capture the behavior of networks of interconnected neurons, and the second part applies these principles to understand the cognitive functions listed above. 6 | 7 | Copyright © 2024 Randall C. O'Reilly, Yuko Munakata, Michael J. Frank, Thomas E. Hazy, and Contributors 8 | 9 | All rights reserved. Licensed under [CC-BY-4.0](https://github.com/CompCogNeuro/book/blob/main/LICENSE). 10 | 11 | For download links and more information, see the [book website](https://compcogneuro.org/book). 12 | -------------------------------------------------------------------------------- /chapter-01.md: -------------------------------------------------------------------------------- 1 | --- 2 | bibfile: ccnlab.bib 3 | --- 4 | 5 | # Introduction {#sec:ch-intro} 6 | 7 | You are about to embark on one of the most fascinating scientific journeys possible: inside your own brain! We start this journey by understanding what individual **neurons** (Chapter 2) in your neocortex do with the roughly 10,000 synaptic input signals that they receive from other neurons. The **neocortex** is the most evolutionarily recent part of the brain, which is also most enlarged in humans, and is where most of your thinking takes place. The numbers of neurons and synapses between neurons in the neocortex are astounding: roughly 20 billion neurons, each of which is interconnected with roughly 10,000 others. That is several times more neurons than people on earth. And each neuron is far more social than we are as people --- estimates of the size of stable human social networks are only around 150-200 people, compared to the 10,000 for neurons. 8 | 9 | We've got a lot going on under the hood. At these scales, the influence of any one neuron on any other is relatively small. We'll see that these small influences can be shaped in powerful ways through *learning mechanisms* (Chapter 4), to achieve complex and powerful forms of information processing. And this information processing prowess does not require much complexity from the individual neurons themselves --- fairly simple forms of information integration both accurately describe the response properties of actual neocortical neurons, and enable sophisticated information processing at the level of aggregate neural *networks* (Chapter 3). 10 | 11 | After developing an understanding of these basic neural information processing mechanisms in Part I of this book, we continue our journey in Part II by exploring many different aspects of human thought (cognition), including perception and attention (Chapter 6), learning and memory (Chapter 7), motor control and reinforcement learning (Chapter 8), executive function (Chapter 9), and language (Chapter 10). Amazingly, all these seemingly different cognitive functions can be understood using the small set of common neural mechanisms developed in Part I. In effect, our neocortex is a fantastic form of silly putty, which can be molded by the learning process to take on many different cognitive tasks. For example, we will find striking similarities across different brain areas and cognitive functions --- the development of primary visual cortex turns out to tell us a lot about the development of rich semantic knowledge of word meanings! 12 | 13 | ## Some Phenomena We'll Explore 14 | 15 | Here is a list of some of the cognitive neuroscience phenomena we'll explore in Part II of the book: 16 | 17 | * **Vision:** We can effortlessly recognize countless people, places, and things. Why is this so hard for robots? We will explore this issue in a network that views natural scenes (mountains, trees, etc.), and develops brain-like ways of encoding them using principles of learning. 18 | 19 | * **Attention:** Where's Waldo? We'll see in a model how two visual processing pathways work together to help focus our attention in different locations in space (whether we are searching for something or just taking things in), and why damage to one of these pathways leads people to ignore half of space. 20 | 21 | * **Episodic memory:** How can damage to a small part of our brain cause amnesia? We'll see how in a model that replicates the structure of the hippocampus. This model provides insight into why the rest of the brain isn't well-suited to take on the job of forming new episodic memories. 22 | 23 | * **Dopamine and Reward:** Why do we get bored with things so quickly? Because our dopamine system is constantly adapting to everything we know, and only gives us rewards when something new or different occurs. We'll see how this all happens through interacting brain systems that drive phasic dopamine release. 24 | 25 | * **Task directed behavior:** How do we stay focused on tasks that we need to get done or things that we need to pay attention to, in the face of an ever-growing number of distractions (like email, text messages, and tweets)? We'll explore this issue through a network that simulates the "executive" part of the brain, the prefrontal cortex. We will see how this area is uniquely-suited to protect us from distraction, and how this can change with age. 26 | 27 | * **Meaning:** ["A rose is a rose is a rose."](https://en.wikipedia.org/wiki/Rose_is_a_rose_is_a_rose_is_a_rose) But how do we know what a rose is in the first place? We'll explore this through a network that "reads" every paragraph in a textbook, and acquires a surprisingly good semantic understanding by noting which words tend to be used together or in similar contexts. 28 | 29 | * **Reading:** What causes dyslexia, and why do people who have it vary so much in their struggles with reading? We'll explore these issues in a network that learns to read and pronounce nearly 3,000 English words, and generalizes to novel nonwords (e.g., "mave" or "nust") just like people do. We'll see why damaging the network in different ways simulates various forms of dyslexia. 30 | 31 | ## The Computational Approach 32 | 33 | An important feature of our journey through the brain is that we use the vehicle of *computer models* to understand cognitive neuroscience (i.e., *Computational Cognitive Neuroscience*). These computer models enrich the learning experience in important ways --- we routinely hear from our students that they didn't really understand anything until they pulled up the computer model and played around with it for a few hours. Being able to manipulate and visualize the brain using a powerful 3D graphical interface brings abstract concepts to life, and enables many experiments to be conducted easily, cleanly, and safely in the comfort of your own laptop. This stuff is fun, like a video game --- think "sim brain", as in the popular "sim city" game from a few years ago. 34 | 35 | At a more serious level, the use of computer models to understand how the brain works has been a critical contributor to scientific progress in this area over the past few decades. A key advantage of computer modeling is its ability to wrestle with complexity that often proves daunting to otherwise unaided human understanding. How could we possibly hope to understand how billions of neurons interacting with 10's of thousands of other neurons produce complex human cognition, just by talking in vague verbal terms, or simple paper diagrams? Certainly, nobody questions the need to use computer models in climate modeling, to make accurate predictions and understand how the many complex factors interact with each other. The situation is only more dire in cognitive neuroscience. 36 | 37 | Nevertheless, in all fields where computer models are used, there is a fundamental distrust of the models. They are themselves complex, created by people, and have no necessary relationship to the real system in question. How do we know these models aren't just completely made-up fantasies? The answer seems simple: the models must be constrained by data at as many levels as possible, and they must generate predictions that can then be tested empirically. In what follows, we discuss different approaches that people might take to this challenge --- this is intended to give a sense of the scientific approach behind the work described in this book --- as a student this is perhaps not so relevant, but it might help give some perspective on how science really works. 38 | 39 | In an ideal world, one might imagine that the neurons in the neural model would be mirror images of those in the actual brain, replicating as much detail as is possible given the technical limitations for obtaining the necessary details. They would be connected exactly as they are in the real brain. And they would produce detailed behaviors that replicate exactly how the organism in question behaves across a wide range of different situations. Then you would feel confident that your model is sufficiently "real" to trust some of its predictions. 40 | 41 | But even if this were technically feasible, you might wonder whether the resulting system would be any more comprehensible than the brain itself! In other words, we would only have succeeded in transporting the fundamental mysteries from the brain into our model, without developing any actual understanding about how the thing really works. From this perspective, the most important thing is to develop *the simplest possible model that captures the most possible data* --- this is basically the principle of *Ockham's razor*, which is widely regarded as a central principle for all scientific theorizing. 42 | 43 | In some cases, it is easy to apply this razor to cut away unnecessary detail. Certainly many biological properties of neurons are irrelevant for their core information processing function (e.g., cellular processes that are common to all biological cells, not just neurons). But often it comes down to a judgment call about what phenomena you regard as being important, which will vary depending on the scientific questions being addressed with the model. 44 | 45 | The approach taken for the models in this book is to find some kind of happy (or unhappy) middle ground between biological detail and cognitive functionality. This middle ground is unhappy to the extent that researchers concerned with either end of this continuum are dissatisfied with the level of the models. Biologists will worry that our neurons and networks are overly simplified. Cognitive psychologists will be concerned that our models are too biologically detailed, and they can make much simpler models that capture the same cognitive phenomena. We who relish this "golden middle" ground are happy when we've achieved important simplifications on the neural side, while still capturing important cognitive phenomena. This level of modeling explores how consideration of neural mechanisms inform the workings of the mind, and reciprocally, how cognitive and computational constraints afford a richer understanding of the problems these mechanisms evolved to solve. It can thus make predictions for how a cognitive phenomenon (e.g., memory interference) is affected by changes at the neural level (due to disease, pharmacology, genetics, or similarly due to changes in the cognitive task parameters). The model can then be tested, falsified and refined. In this sense, a model of cognitive neuroscience is just like any other 'theory', except that it is explicitly specified and formalized, forcing the modeler to be accountable for their theory if/when the data don't match up. Conversely, models can sometimes show that when an existing theory is faced with challenging data, the theory may hold up after all due to a particular dynamic that may not be considered from verbal theorizing. 46 | 47 | Ultimately, it comes down to aesthetic or personality-driven factors, which cause different people to prefer different overall strategies to computer modeling. Each of these different approaches has value, and science would not progress without them, so it is fortunate that people vary in their personalities so different people end up doing different things. Some people value simplicity, elegance, and cleanliness most highly --- these people will tend to favor abstract mathematical (e.g., Bayesian) cognitive models. Other people value biological detail above all else, and don't feel very comfortable straying beyond the most firmly established facts --- they will prefer to make highly elaborated individual neuron models incorporating everything that is known. To live in the middle, you need to be willing to take some risks, and value most highly the process of *emergence*, where complex phenomena can be shown to emerge from simpler underlying mechanisms. 48 | 49 | The criteria for success here are a bit murkier and subjective --- basically it boils down to whether the model is sufficiently simple to be comprehensible, but not so simple as to make its behavior trivial or otherwise so fully transparent that it doesn't seem to be doing you any good in the first place. One last note on this issue is that the different levels of models are not mutually exclusive. Each of the low level biophysical and high level cognitive models have made enormous contributions to understanding and analysis in their respective domains (much of which is a basis for further simplification or elaboration in the book). In fact, much ground can be (and to some extent already has been) gained by attempts to understand one level of modeling in terms of the other. At the end of the day, linking from molecule to mind spans multiple levels of analysis, and like studying the laws of particle physics to planetary motion, require multiple formal tools. 50 | 51 | ## Emergent Phenomena 52 | 53 | What makes something a satisfying scientific explanation? A satisfying answer is that you can explain a seemingly complex phenomenon in terms of simpler underlying mechanisms, that interact in specific ways. The classic scientific process of *reductionism* plays a critical role here, where the complex system is reduced to simpler parts. However, one also needs to go in the opposite, oft-neglected direction, *reconstructionism*, where the complex system is actually reconstructed from these simpler parts. Often the only way to practically achieve this reconstruction is through computational modeling. The result is an attempt to capture the essence of emergence. 54 | 55 | ![The principle of *emergence*, simply illustrated. The gears on the left do not interact, and nothing interesting happens. However, on the right, the interaction between the gears produces interesting, useful phenomena that *cannot* be reduced to the individual gears *separately*. For example, the little gear will spin faster, but the larger one will have higher torque at its axel --- these properties would be entirely different if either gear interacted with a different sized gear. Furthermore, the material that the gear is made from really doesn't matter very much --- the same basic behavior would be produced by plastic, metal, wood, etc. Thus, even in this simple case, there is something just slightly magical and irreducible going on --- when two gears get together, something emerges that is more than the sum of the parts, and exists in a way independent of the parts, even while being entirely dependent on actually *having* those parts to make it happen. This is a good analogy for the relationship between the mind and the brain, and computer models can capture many complex interactions between neurons in the brain, and reveal nonobvious kinds of emergence.](figures/fig_gears.png){#fig:fig-gears width=75% } 56 | 57 | Emergence can be illustrated in a very simple physical system, two interacting gears, as shown in [@fig:fig-gears]. It is not mysterious or magical. On the other hand, it really is. You can make the gears out of any kind of sufficiently hard material, and they will still work. There might be subtle factors like friction and durability that vary. But over a wide range, it doesn't matter what the gears are made from. Thus, there is a level of *transcendence* that occurs with emergence, where the behavior of the more complex interacting system does not depend on many of the detailed properties of the lower level parts. In effect, the interaction itself is what matters, and the parts are mere place holders. Of course, they have to be there, and meet some basic criteria, but they are nevertheless replaceable. 58 | 59 | Taking this example into the domain of interest here, does this mean that we can switch out our biological neurons for artificial ones, and everything should still function the same, *as long as we capture the essential interactions in the right way?* Some of us believe this to be the case, and that when we finally manage to put enough neurons in the right configuration into a big computer simulation, the resulting brain will support consciousness and everything else, just like the ones in our own heads. One interesting further question arises: how important are all the interactions between our physical bodies and the physical environment? There is good reason to believe that this is critical. Thus, we'll have to put this brain in a robot. Or perhaps more challengingly, in a virtual environment in a virtual reality, still stuck inside the computer. It will be fascinating to ponder this question on your journey through the simulated brain... 60 | 61 | ## Why Should We Care about the Brain? 62 | 63 | One of the things you'll discover on this journey is that *Computational Cognitive Neuroscience is hard*. There is a lot of material at multiple levels to master. We get into details of ion channels in neurons, names of pathways in different parts of the brain, effects of lesions to different brain areas, and patterns of neural activity, on top of all the details about behavioral paradigms and reaction time patterns. Wouldn't it just be a lot simpler if we could ignore all these brain details, and just focus on what we *really* care about --- how does cognition itself work? By way of analogy, we don't need to know much of anything about how computer hardware works to program in Visual Basic or Python, for example. Vastly different kinds of hardware can all run the same programming languages and software. Can't we just focus on the *software* of the mind and ignore the *hardware*? 64 | 65 | Exactly this argument has been promulgated in many different forms over the years, and indeed has a bit of a resurgence recently in the form of abstract Bayesian models of cognition. David Marr was perhaps the most influential in arguing that one can somewhat independently examine cognition at three different levels [@Marr77]: 66 | 67 | * **Computational** --- what computations are being performed? What information is being processed? 68 | 69 | * **Algorithmic** --- how are these computations being performed, in terms of a sequence of information processing steps? 70 | 71 | * **Implementational** --- how does the hardware actually implement these algorithms? 72 | 73 | This way of dividing up the problem has been used to argue that one can safely ignore the implementation (i.e., the brain), and focus on the computational and algorithmic levels, because, like in a computer, the hardware really doesn't matter so much. 74 | 75 | However, the key oversight of this approach is that the reason hardware doesn't matter in standard computers is that *they are all specifically designed to be functionally equivalent in the first place!* Sure, there are lots of different details, but they are all implementing a basic serial Von Neumann architecture. What if the brain has a vastly different architecture, which makes some algorithms and computations work extremely efficiently, while it cannot even support others? Then the implementational level would matter a great deal. 76 | 77 | There is every reason to believe that this is the case. The brain is *not* at all like a general purpose computational device. Instead, it is really a custom piece of hardware that implements a very specific set of computations in massive parallelism across its 20 billion neurons. In this respect, it is much more like the specialized graphics processing units (GPUs) in modern computers, which are custom designed to efficiently carry out in massive parallelism the specific computations necessary to render complex 3D graphics. More generally, the field of computer science is discovering that parallel computation is exceptionally difficult to program, and one has to completely rethink the algorithms and computations to obtain efficient parallel computation. Thus, the hardware of the brain matters a huge amount, and provides many important clues as to what kind of algorithms and computations are being performed. 78 | 79 | Historically, the "ignore the brain" approaches have taken an interesting trajectory. In the 1960's through the early 1990's, the dominant approach was to assume that the brain actually operates much like a standard computer, and researchers tended to use concepts like logic and symbolic propositions in their cognitive models. Since then, a more statistical metaphor has become popular, with the Bayesian probabilistic framework being widely used in particular. This is an advance in many respects, as it emphasizes the graded nature of information processing in the brain (e.g., integrating various graded probabilities to arrive at an overall estimate of the likelihood of some event), as contrasted with hard symbols and logic, which didn't seem to be a particularly good fit with the way that many (though not all!) aspects of cognition actually operate. 80 | 81 | However, the actual mathematics of Bayesian probability computations are not a particularly good fit to how the brain operates at the neural level, and much of this research operates without much consideration for how the brain actually functions. Instead, a version of Marr's computational level has been adopted, by assuming that whatever the brain is doing, it must be at least close to optimal, and Bayesian models can often tell us how to optimally combine uncertain pieces of information. Regardless of the validity of this optimality assumption, it is definitely useful to know what the optimal computations are for given problems, so this approach certainly has a lot of value in general. However, optimality is typically conditional on a number of assumptions, and it is often difficult to decide among these different assumptions. 82 | 83 | ![Models that are relatively unconstrained, e.g., by not addressing biological constraints, or detailed behavioral data, are like jigsaw puzzles of a featureless blue sky --- very hard to solve --- you just don't have enough clues to how everything fits together.](figures/fig_brain_puzzle_blue_sky_pieces.png){#fig:fig-puzzle-blue-sky width=65% } 84 | 85 | If you really want to know for sure how the brain is actually producing cognition, clearly you need to know how the brain actually functions. Yes, this is hard. But it is not impossible, and the state of neuroscience these days is such that there is a wealth of useful information to inform all manner of insights into how the brain actually works. It is like working on a jigsaw puzzle --- the easiest puzzles are full of distinctive textures and junk everywhere, so you can really see when the pieces fit together [@fig:fig-puzzle-clutter]). The rich tableau of neuroscience data provides all this distinctive junk to constrain the process of puzzling together cognition. In contrast, abstract, purely cognitive models are like a jigsaw puzzle with only a big featureless blue sky ([@fig:fig-puzzle-blue-sky]). You only have the logical constraints of the piece shapes, which are all highly similar and difficult to discriminate. It takes forever. 86 | 87 | ![Adding constraints from biology and detailed consideration of behavior provide a rich set of clues for figuring out how to solve the puzzle of the brain!](figures/fig_brain_puzzle_behav_bio_data.png){#fig:fig-puzzle-clutter width=65% } 88 | 89 | A couple of the most satisfying instances of all the pieces coming together to complete a puzzle include: 90 | 91 | * The detailed biology of the hippocampus, including high levels of inhibition and broad diffuse connectivity, fit together with its unique role in rapidly learning new episodic information, and the remarkable data from patient HM who had his hippocampus resected to prevent intractable epilepsy. Through computational models in Chapter 8 (Memory), we can see that these biological details produce high levels of *pattern separation* which keep memories highly distinct, and thus enable rapid learning without creating catastrophic levels of interference. 92 | 93 | * The detailed biology of the connections between dopamine, basal ganglia, and prefrontal cortex fit together with the computational requirements for making decisions based on prior reward history, and learning what information is important to hold on to, versus what can be ignored. Computational models in Chapter 10 (Executive Function) show that the dopamine system can exhibit a kind of time travel needed to translate later utility into an earlier decision of what information to maintain, and those in Chapter 7 (Motor) show that the effects of dopamine on the basal ganglia circuitry are just right to facilitate decision making based on both positive and negative outcomes. And the interaction between the basal ganglia and the prefrontal cortex enables basal ganglia decisions to influence what is maintained and acted upon in the prefrontal cortex. There are a lot of pieces here, but the fact that they all fit together so well into a functional model --- and that many aspects of them have withstood the test of direct experimentation --- makes it that much more likely that this is really what is going on. 94 | 95 | ## AI, ML, and Neuroscience 96 | 97 | The core material in this textbook has been around for 20 years now, since 2000 [@OReillyMunakata00], and the overall popularity of neural network models has fluctuated considerably during that time. Currently, as of the 4th edition in 2020, there has been a major resurgence of interest in using "deep" neural networks for artificial intelligence (AI) and machine learning (ML) [@LeCunBengioHinton15; @Schmidhuber15a]. These models are now powering practical, though still very narrow, applications on modern smart phones, outperforming other techniques in a wide range of ML competitions [@KrizhevskySutskeverHinton12], and beating humans at their most cherished games [@SilverSchrittwieserSimonyanEtAl17]. Interestingly, the core principles powering these new AI models are consistent with many of those based on how the brain functions, including the error backpropagation learning algorithm which we discuss in detail in the *Learning* Chapter. 98 | 99 | However, these new AI models also include many mechanisms that are *not* consistent with the underlying biology, and the performance-based goals for these models are often at odds with the more purely *scientific* goals of understanding how the brain works. Thus, the material in this text is complementary to AI / ML approaches. Furthermore, it is still the case that the human brain is the undisputed champion at *general intelligence:* the ability to do many different tasks reasonably well, and learn with relatively minimal levels of explicit instruction to perform new tasks. 100 | 101 | Interestingly, one core feature of the human brain, which is present in the models developed in this book, but largely absent in current AI models, is the pervasive *bidirectional connectivity* among neurons: neurons in the brain are *mutually* interacting, highly "social" little cells. Furthermore, this bidirectional connectivity or *recurrence* has been tied to *consciousness* [@Lamme06], thus suggesting the interesting possibility that the human capacity for consciousness, based on this bidirectional connectivity, may be a key element of our general intelligence capacity. In particular, consciousness may enable flexible access to knowledge by many different routes, as knowledge can "float" across domains in the "plenary" *global workspace* of consciousness [@Baars88], and we can directly, *consciousiously* manipulate and control our knowledge to direct it toward desired cognitive goals. 102 | 103 | The models in this textbook provide a foundation for understanding this kind of flexible cognitive function, and current research is directly focused on exploring these ideas further, so that ultimately we may yet understand more of the deep secrets of the human mind! 104 | 105 | ## How to Read this Book 106 | 107 | This book is intended to accommodate many different levels of background and interests. The main chapters are relatively short, and provide a high-level introduction to the major themes. There will be an increasing number of detailed subsections added over time, to support more advanced treatment of specific issues. The ability to support these multiple levels of readers is a major advantage of the wiki format. We also encourage usage of this material as an adjunct for other courses on related topics. The simulation models can be used by themselves in many different courses. 108 | 109 | Due to the complexity and interconnected nature of the material (mirroring the brain itself), it may be useful to revisit earlier chapters after having read later chapters. Also, we strongly recommend reading Chapter 5 (Brain Areas) chapter *now*, and then re-reading it in its regular sequence after having made it all the way through Part I. It provides a nice high-level summary of functional brain organization, that bridges the two parts of the book, and gives an overall roadmap of the content we'll be covering. Some of it won't make as much sense until after you've read Part I, but doing a quick first read now will provide a lot of useful perspective. 110 | 111 | ## External Resources 112 | 113 | * [Gary Cottrell's solicited compilation of important computational modeling papers](https://cseweb.ucsd.edu/users/gary/CogSciLiterature.html) 114 | -------------------------------------------------------------------------------- /chapter-05.md: -------------------------------------------------------------------------------- 1 | --- 2 | bibfile: ccnlab.bib 3 | --- 4 | 5 | # Part II: Brain Areas {#sec:ch-areas} 6 | 7 | In Part I of this book, we have developed a toolkit of basic neural mechanisms, going from the activation dynamics of individual neurons, to networks of neurons, and the learning mechanisms that configure them in both self-organizing and error-driven ways. At this start of Part II, we begin the transition to exploring a wide range of cognitive phenomena. As an important foundational step along this path, this chapter attempts to provide a big picture view of the overall functional organization of the brain, in a relatively non-controversial way that is roughly meant to correspond to what is generally agreed upon in the literature. This should help you understand at a broad level how different brain areas work together to perform different cognitive functions, and situate the more specific models in the subsequent chapters into a larger overall framework. 8 | 9 | We proceed in the same sequence as the subsequent chapters, starting with basic perceptual systems, and then proceeding to explore different forms of learning and memory (including the role of the hippocampus in episodic memory). We then turn to basic motor systems, and build upon them to examine executive function. Finally, we build upon and extend the functionality of all of these cognitive systems as we examine language. 10 | 11 | As usual, we begin with a basic foundation in biology: the gross anatomy of the brain. 12 | 13 | ## Navigating the Functional Anatomy of the Brain 14 | 15 | ![Gross anatomy of the brain. Left panel shows the major lobes of the outer neocortex layer of the brain, and right panel shows some of the major brain areas internal to the neocortex.](figures/fig_brain_and_lobes.png){#fig:fig-brain-anatomy width=90% } 16 | 17 | [@fig:fig-brain-anatomy] shows the "gross" (actually quite beautiful and amazing!) anatomy of the brain. The outer portion is the "wrinkled sheet" (upon which our thoughts rest) of the **neocortex**, showing all of the major lobes. This is where most of our complex cognitive function occurs, and what we have been focusing on to this point in the text. The rest of the brain lives inside the neocortex, with some important areas shown in the figure. These are generally referred to as **subcortical** brain areas, and we include some of them in our computational models, including: 18 | 19 | * **Hippocampus** --- this brain area is actually an "ancient" form of cortex called "archicortex", and we'll see in the *Memory* Chapter how it plays a critical role in learning new "everyday" memories about events and facts (called *episodic* memories). 20 | 21 | * **Amygdala** --- this brain area is important for recognizing emotionally salient stimuli, and alerting the rest of the brain about them. We'll explore it in the *Motor Control and Reinforcement Learning* Chapter, where it plays an important role in reinforcing motor (and cognitive) actions based on reward (and punishment). 22 | 23 | * **Cerebellum** --- this massive brain structure contains 1/2 of the neurons in the brain, and plays an important role in motor coordination. It is also active in most cognitive tasks, but understanding exactly what its functional role is in cognition remains somewhat elusive. We'll explore it in the *Motor Control and Reinforcement Learning* Chapter. 24 | 25 | * **Thalamus** --- provides the primary pathway for sensory information on its way to the neocortex, and is also likely important for attention, arousal, and other modulatory functions. We'll explore the role of visual thalamus in the *Perception and Attention* Chapter and of motor thalamus in the *Motor Control and Reinforcement Learning* Chapter. 26 | 27 | * **Basal Ganglia** --- this is a collection of subcortical areas that plays a critical role in the *Motor Control and Reinforcement Learning* Chapter, and also in *Executive Function* Chapter. It helps to make the final "Go" call on whether (or not) to execute particular actions that the cortex 'proposes', and whether or not to update cognitive plans in the prefrontal cortex. Its policy for making these choices is learned based on their prior history of reinforcement/punishment. 28 | 29 | ![Terminology for referring to different parts of the brain --- for everything except lateral and medial, three different terms for the same thing are given.](figures/fig_location_terms.png){#fig:fig-location-terms width=30% } 30 | 31 | [@fig:fig-location-terms] shows the terminology that anatomists use to talk about different parts of the brain --- it is a good idea to get familiar with these terms --- we'll put them to good use right now. 32 | 33 | ![Brodmann's numbering system for the different areas of the neocortex, based on anatomical distinctions such as the thickness of different cortical layers, as we discussed in the Networks Chapter. These anatomical distinctions are remarkably well correlated with the functional differences in what different brain areas do.](figures/fig_brodmann_areas_color.png){#fig:fig-brodmann-areas-color width=50% } 34 | 35 | ![Color delineated 3D map of Brodmann areas on the external cortical surface. *Top:* anterior view. *Bottom:* posterior view.](figures/fig_brodmann_areas_3d_color.png){#fig:fig-brodmann-areas-3d-color width=30% } 36 | 37 | ![Summary of functions of cortical lobes --- see text for details.](figures/fig_cortex_lobes.png){#fig:fig-cortex-lobes width=50% } 38 | 39 | [@fig:fig-brodmann-areas-color; @fig:fig-brodmann-areas-3d-color] show more detail on the structure of the neocortex, in terms of **Brodmann areas** --- these areas were identified by Korbinian Brodmann on the basis of anatomical differences (principally the differences in thickness of different cortical layers, which we covered in the *Networks* Chapter). We won't refer too much to things at this level of detail, but learning some of these numbers is a good idea for being able to read the primary literature in cognitive neuroscience. Here is a quick overview of the functions of the cortical lobes ([@fig:fig-cortex-lobes]): 40 | 41 | * **Occipital lobe** --- this contains the **primary visual cortex (V1)** (Brodmann's area 17 or BA17), located at the very back tip of the neocortex, and higher-level visual areas that radiate out (forward) from it. Clearly, its main function is in visual processing. 42 | 43 | * **Temporal lobe** --- departing from the occipital lobe, the **what** pathway of visual processing dives down into the **inferior temporal cortex (IT)**, where visual objects are recognized. Meanwhile, the superior temporal cortex contains the **primary auditory cortex (A1)**, and associated higher-level auditory and **language-processing** areas. Thus, the temporal lobes (one on each side) are where the visual appearance of objects gets translated into verbal labels (and vice-versa), and also where we learn to read. The most anterior region of the temporal lobes appears to be important for **semantic knowledge** --- all your high-level understanding of things like lawyers and government and all that good stuff you learn in school. The **medial temporal lobe** (MTL) area transitions into the hippocampus, and areas here play an increasingly important role in storing and retrieving memories of life events (**episodic memory**). When you are doing rote memorization without deeper semantic learning, the MTL and hippocampus are hard at work. Eventually, as you learn things more deeply and systematically, they get encoded in the anterior temporal cortex (and other brain areas too). In summary, the temporal lobes contain a huge amount of the stuff that we are consciously aware of --- facts, events, names, faces, objects, words, etc. One broad characterization is that temporal cortex is good at **categorizing** the world in myriad ways. 44 | 45 | * **Parietal lobe** --- in contrast to the temporal lobe, the parietal lobe is much murkier and subconscious. It is important for encoding spatial locations (i.e., the **where** pathway, in complement to the IT what pathway), and damage to certain parts of it gives rise to the phenomenon of hemispatial neglect --- people just forget about an entire half of space! But its functionality goes well beyond mere spatial locations. It is important for encoding **numbers, mathematics, abstract relationships**, and many other "smart" things. At a more down-to-earth level, the parietal cortex provides the major pathway where visual information can **guide motor actions**, leading it to be characterized as the **how** pathway. It also contains the **primary somatosensory cortex (S1)**, which is important for guiding and informing motor actions as well. In some parts of the parietal cortex, neurons serve to translate between different **frames of reference**, for example converting spatial locations on the body (from somatosensation) to visual coordinates. And visual information can be encoded in terms of the patterns of activity on the retina (retinotopic coordinates), or head, body, or environment-based reference frames. One broad characterization of parietal cortex is that it is specialized for processing **metrical** information --- things that vary along a continuum, in direct contrast with the discrete, categorical nature of temporal lobe. A similar distinction is popularly discussed in terms of left vs. right sides of the brain, but the evidence for this in terms of temporal vs. parietal is stronger overall. 46 | 47 | * **Frontal lobe** --- this starts at the posterior end with the **primary motor cortex (M1)**, and moving forward, there is a hierarchy of higher-levels of motor control, from low level motor control in M1 and supplementary motor areas (SMA), up to higher-level action sequences and contingent behavior encoded in premotor areas (higher motor areas). Beyond this is the **prefrontal cortex (PFC)**, known as the brain's **executive** --- this is where all the high-level shots are called, where your big plans are sorted out and influenced by basic motivations and emotions, to determine what you really should do next. The PFC also has a posterior-anterior organization, with more anterior areas encoding higher-level, longer-term plans and goals. The most anterior area of PFC (the **frontal pole**) seems to be particularly important for the most abstract, challenging forms of cognitive reasoning --- when you're really trying hard to figure out a puzzle, or sort through those tricky questions on the GRE or an IQ test. The medial and ventral regions of the frontal cortex are particularly important for **emotion and motivation** --- for example the **orbital frontal cortex (OFC)** seems to be important for maintaining and manipulating information about how rewarding a given stimulus or possible outcome might be (it receives a strong input from the Amygdala to help it learn and represent this information). The **anterior cingulate cortex (ACC)** is important for encoding the consequences of your actions, including the difficulty, uncertainty, or likelihood of failure associated with prospective actions in the current state (it lights up when you look down that double-black diamond run at the ski area!). Both the OFC and the ACC can influence choices via interactions with other frontal motor plan areas, and also via interactions with the basal ganglia. The **ventromedial PFC (VMPFC)** interacts with a lot of subcortical areas, to control basic bodily functions like heart rate, breathing, and neuromodulatory areas that then influence the brain more broadly (e.g., the ventral tegmental area (VTA) and locus coeruleus (LC), which release **dopamine** and **norepinephrine**, both of which have broad effects all over the cortex, but especially back in frontal cortex). The biggest mystery about the frontal lobe is how to understand how it does all of these amazing things, without using terms like "executive", because we're pretty sure you don't have a little guy in a pinstripe suit sitting in there. It is all just neurons! 48 | 49 | ## Comparing and Contrasting Major Brain Areas 50 | 51 | | | *Learning Signal* | | | *Dynamics* | | | 52 | |---------------|-------------------|------------|----------|------------|------------|-----------| 53 | | Area | Reward | Error | Self Org | Separator | Integrator | Attractor | 54 | |---------------|-------------------|------------|----------|------------|------------|-----------| 55 | | Basal Ganglia | +++ | --- | --- | ++ | - | --- | 56 | | Cerebellum | --- | +++ | --- | +++ | --- | --- | 57 | | Hippocampus | + | + | +++ | +++ | --- | +++ | 58 | | Neocortex | ++ | +++ | ++ | --- | +++ | +++ | 59 | 60 | Table: Comparison of learning mechanisms and activity/representational dynamics across four primary areas of the brain. `+++` means that the area definitely has given property, with fewer `+` indicating less confidence in and/or importance of this feature. `---` means that the area definitely does not have the given property, again with fewer `-` indicating lower confidence or importance. {#tbl:table-learning} 61 | 62 | [@tbl:table-learning] shows a comparison of four major brain areas according to the learning rules and activation dynamics that they employ. We haven't yet discussed reward-driven learning. We'll do so in the *Motor Control and Reinforcement Learning* Chapter, but for now think of it as learning to predict which pattern of activation will maximize a reward signal (e.g., dopamine level). 63 | 64 | The evolutionarily older areas of the basal ganglia, cerebellum, and hippocampus employ a separating form of activation dynamics, meaning that they tend to make even somewhat similar inputs map onto more separated patterns of neural activity within the structure. This is a very conservative, robust strategy akin to memorizing specific answers to specific inputs --- it is likely to work OK, even though it is not very efficient, and does not generalize to new situations very well. Each of these structures can be seen as optimizing a different form of learning within this overall separating dynamic. The basal ganglia are specialized for learning on the basis of reward expectations and outcomes. The cerebellum uses a simple yet effective form of error-driven learning (basically the delta rule as discussed in the *Learning* Chapter). And the hippocampus relies more on hebbian-style self-organizing learning. Thus, the hippocampus is constantly encoding new episodic memories regardless of error or reward (though these can certainly modulate the rate of learning, as indicated by the weaker + signs in the table), while the basal ganglia is learning to select motor actions on the basis of potential reward or lack thereof (and is also a control system for regulating the timing of action selection), while the cerebellum is learning to swiftly perform those motor actions by using error signals generated from differences in the sensory feedback relative to the motor plan. Taken together, these three systems are sufficient to cover the basic needs of an organism to survive and adapt to the environment, at least to some degree. 65 | 66 | The hippocampus introduces one critical innovation beyond what is present in the basal ganglia and cerebellum: it has attractor dynamics. Specifically the recurrent connections between CA3 neurons are important for retrieving previously-encoded memories, via pattern completion as we explored in the *Networks* Chapter. The price for this innovation is that the balance between excitation and inhibition must be precisely maintained, to prevent epileptic activity dynamics. Indeed, the hippocampus is the single most prevalent source of epileptic activity, in people at least. 67 | 68 | Against this backdrop of evolutionarily older systems, the neocortex represents a few important innovations. In terms of activation dynamics, it builds upon the attractor dynamic innovation from the hippocampus (appropriately so, given that hippocampus represents an ancient "proto" cortex), and adds to this a strong ability to develop representations that integrate across experiences to extract generalities, instead of always keeping everything separate all the time. The cost for this integration ability is that the system can now form the wrong kinds of generalizations, which might lead to bad overall behavior. But the advantages apparently outweigh the risks, by giving the system a strong ability to apply previous learning to novel situations. In terms of learning mechanisms, the neocortex employs a solid blend of all three major forms of learning, integrating the best of all the available learning signals into one system. 69 | 70 | Each chapter in the remainder of Part II focuses on a different cognitive function. Below we preview each chapter. 71 | 72 | ## Perception and Attention: What vs. Where 73 | 74 | ![Hierarchy of visual detectors of increasing complexity achieves sophisticated perceptual categorization, with the higher levels being able to recognize thousands of different objects, people, etc.](figures/fig_category_hierarch_dist_reps.png){#fig:fig-category_hierarch_dist_reps-5 width=100% } 75 | 76 | ![Updated and simplified version of Felleman & Van Essen's (1991) diagram of the anatomical connectivity of visual processing pathways, starting with primary visual cortex (V1) and on up. The blue-shaded areas comprise the ventral *What* pathway, and green-shaded are the dorsal *Where*. Reproduced from Markov et al., (2014).](figures/fig_markov_et_al_cort_hier.png){#fig:fig-markov_cort_hier-5 width=50% } 77 | 78 | ![Division of *What* vs *Where* (ventral vs. dorsal) pathways in visual processing, based on the classic Ungerlieder and Mishkin (1982) framework.](figures/fig_vis_system_bio.png){#fig:fig-vis_system_bio-5 width=50% } 79 | 80 | The perceptual system provides an excellent example of the power of hierarchically organized layers of neural detectors, as we discussed in the *Networks* Chapter. [@fig:fig-category_hierarch_dist_reps-5] summarizes this process, with associated cortical areas noted below each stage of processing. [@fig:fig-markov_cort_hier-5] shows the current best estimate of the actual anatomical connectivity patterns of all of the major visual areas [@MarkovVezoliChameauEtAl14; @FellemanVanEssen91], showing that information really is processed in a hierarchical fashion in the brain (although there are many interconnections outside of a strict hierarchy as well). [@fig:fig-vis_system_bio-5] puts these areas into their anatomical locations, showing more clearly the **what vs where** (**ventral vs dorsal**) split in visual processing [@UngerleiderMishkin82]. Here is a quick summary of the flow of information up the **what** side of the visual pathway (pictured on the left side of [@fig:fig-markov_cort_hier-5]): 81 | 82 | * **V1** --- primary visual cortex, which encodes the image in terms of oriented edge detectors that respond to edges (transitions in illumination) along different angles of orientation. We will see in the *Perception and Attention* Chapter how these edge detectors develop through self-organizing learning, driven by the reliable statistics of natural images. 83 | 84 | * **V2** --- secondary visual cortex, which encodes combinations of edge detectors to develop a vocabulary of intersections and junctions, along with many other basic visual features (e.g., 3D depth selectivity, basic textures, etc), that provide the foundation for detecting more complex shapes. These V2 neurons also encode these features in a broader range of locations, starting a process that ends up with IT neurons being able to recognize an object regardless of where it appears in the visual field (i.e., *invariant* object recognition). 85 | 86 | * **V4** --- detects more complex shape features, over an even larger range of locations (and sizes, angles, etc). 87 | 88 | * **IT-posterior** (PIT) --- detects entire object shapes, over a wide range of locations, sizes, and angles. For example, there is an area near the fusiform gyrus on the bottom surface of the temporal lobe, called the **fusiform face area (FFA)**, that appears especially responsive to faces. As we saw in the *Networks* Chapter, however, objects are encoded in distributed representations over a broad range of areas in IT. 89 | 90 | * **IT-anterior** (AIT) --- this is where visual information becomes extremely abstract and **semantic** in nature --- as shown in the Figure, it can encode all manner of important information about different people, places and things. 91 | 92 | We'll explore a model of invariant object recognition in the *Perception and Attention* Chapter that shows how this deep hierarchy of detectors can develop through learning. The *Language* Chapter builds upon this object recognition process to understand how words are recognized and translated into associated verbal motor outputs during reading, and associated with semantic knowledge as well. 93 | 94 | The **where** aspect of visual processing happens in the parietal cortex, including areas such as middle temporal (MT), ventral intraparietal cortex (VIP), lateral intraparietal cortex (LIP), and medial superior temporal (MST), that are important for processing motion, depth, and other spatial features. As noted above, these areas are also critical for translating visual input into appropriate motor output, leading Goodale and Milner to characterize this as the **how** pathway [@GoodaleMilner92]. In the *Perception and Attention* Chapter, we'll see how this dorsal pathway can interact with the ventral what pathway in the context of visual attention, producing the characteristic effects of parietal damage such as hemispatial neglect, for example. There is also increasing evidence for three distinct parietal-lobe pathways, corresponding to *looking* (lateral intraparietal cortex (LIP) / frontal eye fields (FEF)), *reaching* (VIP / supplementary motor area (SMA)), and *navigating* (medial parietal networks, including posterior cingulate cortex (PCC) and retrosplenial cortex (RSC)) [@KravitzSaleemBakerEtAl11; @RanganathRitchey12]. 95 | 96 | ## Memory: Temporal Cortex and the Hippocampus 97 | 98 | When you think of memory, probably things like "what did I have for dinner last night?" and "how can I remember people's names better?" tend to come to mind. These represent just one category of memory, however. Indeed, memory is ubiquitous in neural networks --- every synapse has the capacity for storing memory, and any given "memory" requires the coordinated actions of millions of such synapses to encode and retrieve. There are many taxonomies of memory, but really the only one you need to know is identical to the functional organization of the brain being provided here. Memory is embedded in every brain area, and the nature of that memory is intimately tied up with what that area does. Motor cortex learns motor memories. Parietal cortex learns things like motor skills --- how to hit a baseball (hint: keep your eye on the ball --- parietal cortex needs visual input!). 99 | 100 | ![The hippocampus sits on "top" of the cortical hierarchy and can encode information from all over the brain, binding it together into an episodic memory.](figures/fig_hippo_mem_formation.png){#fig:fig-hippo_mem_formation-5 width=75% } 101 | 102 | There is one brain area, however, that looms so large in the domain of memory, that we'll spend a while focusing on it. This is the **hippocampus**, which seems to be particularly good at rapidly learning new information, in a way that doesn't interfere too much with previously learned information [@fig:fig-hippo_mem_formation-5]. When you need to remember the name associated with a person you recently met, you're relying on this rapid learning ability of the hippocampus. We'll see that the neural properties of the hippocampal system are ideally suited to producing this rapid learning ability. One key neural property is the use of **extremely sparse representations**, which produce a phenomenon called **pattern separation**, where the neural activity pattern associated with one memory is highly distinct from that associated with other similar memories [@Marr71; @McClellandMcNaughtonOReilly95]. This is what minimizes interference with prior learning --- interference arises as a function of overlap. We'll see how this pattern separation process is complemented by a pattern completion process for recovering memories during retrieval from partial information [@OReillyMcClelland94]. 103 | 104 | We'll also see how the **learning rate** plays a crucial role in learning. Obviously, to learn rapidly, you need a fast learning rate. But what happens with a slow learning rate? Turns out this enables you to integrate across many different experiences, to produce *wisdom* and *semantic knowledge*. This slower learning rate is characteristic of most of the neocortex (it also enables the basal ganglia to learn probabilities of positive and negative outcomes for each action across a range of experience, rather than just their most recent outcomes) [@McClellandMcNaughtonOReilly95]. Interestingly, even with a slow learning rate, neocortex can exhibit measurable effects of a single trial of learning, in the form of **priming effects** and **familiarity signals** that can drive recognition memory (i.e., your ability to recognize something as familiar, without any more explicit episodic memory). This form of recognition memory seems to depend on **medial temporal lobe (MTL)** areas including **perirhinal cortex**. Another form of one trial behavioral learning involves mechanisms that support active maintenance of memories in an attractor state (working memory in the prefrontal cortex). This form of memory does not require a weight change at all, but can nevertheless rapidly influence behavioral performance from one instance to the next. 105 | 106 | ## Motor Control: Parietal and Motor Cortex, Basal Ganglia and Cerebellum 107 | 108 | Carrying the parietal **how** pathway forward, visual information going along the dorsal pathway through the parietal cortex heads directly into the frontal cortex, where it can drive motor neurons in the primary motor cortex, which can directly drive the muscles to produce overt motor actions. This completes the critical sensory-motor loop that lies at the core of all behavior. Motor control also critically involves many subcortical brain areas, including the basal ganglia and cerebellum. The rough division of labor between these areas is: 109 | 110 | * **Neocortex (parietal to frontal)** --- does high-level metrical processing of sensory information, integrating multiple modalities and translating between different reference frames as necessary, to arrive at a **range of possible responses to the current sensory environment.** 111 | 112 | * **Basal Ganglia** --- receives both sensory inputs and the potential responses being "considered" in frontal cortex, and can then trigger a **disinhibitory Go** signal that enables the *best* of the possible actions to get over threshold and actually drive behavior [@Mink96; @Frank05]. This process of **action selection** is shaped by **reinforcement learning** --- the basal ganglia are bathed in **dopamine**, which drives learning in response to rewards and punishments, and also influences the speed of the selection process itself [@SuttonBarto98; @MontagueDayanSejnowski96]. Thus, the basal ganglia selects the action that is most likely to result in reward, and least likely to result in punishment. The **amygdala** plays a key role in driving these dopamine signals in response to sensory cues associated with reward and punishment. 113 | 114 | * **Cerebellum** --- is richly interconnected with the parietal and motor cortex, and it is capable of using a simple yet powerful form of error-driven learning to acquire high-resolution metrical maps between sensory inputs and motor outputs. Thus, it is critical for generating smooth, coordinated motor movements that properly integrate sensory and motor feedback information to move in an efficient and controlled manner. It also likely serves to teach the parietal and motor cortex what it has learned. 115 | 116 | In the *Motor Control and Reinforcement Learning* Chapter, we will see how dopamine signals shape basal ganglia learning and performance in a basic action selection task. Then, we'll explore a fascinating model of cerebellar motor learning in a virtual robot that performs coordinated eye and head movements to fixate objects --- this model shows how the error signals needed for cerebellar learning can arise naturally. 117 | 118 | Interestingly, all of these "low level" motor control systems end up being co-opted by "higher level" executive function systems (e.g., the prefrontal cortex), so although some don't think of motor control as a particularly cognitive domain, it actually provides a solid foundation for understanding some of the highest levels of cognitive function! 119 | 120 | ## Executive Function: Prefrontal Cortex and Basal Ganglia 121 | 122 | ![The *What* vs. *How* distinction for posterior cortex can be carried forward into prefrontal cortex, to understand the distinctive roles of the ventral and dorsal areas of PFC.](figures/fig_cortical_fun_org_tins_acc_ofc.png){#fig:fig-cortical_fun_org-5 width=60% } 123 | 124 | We build upon the motor control functions of frontal cortex and basal ganglia to understand how these two areas interact to support high-level executive function. We also build upon the functional divisions of the posterior cortex to understand how the ventral vs. dorsal areas of prefrontal cortex are functionally organized. [@fig:fig-cortical_fun_org-5] shows an overall schematic for how this occurs. It also illustrates how the lateral surface is more associated with "cold" cognitive function, while the medial surface is more involved in "hot" emotional and motivational processing. 125 | 126 | We'll see how the PFC can provide **top-down cognitive control** over processing in the posterior cortex, with the classic example being the Stroop task. 127 | 128 | Then we'll explore how PFC and BG can interact to produce a dynamically gated working memory system that allows the system to hold multiple pieces of information 'in mind', and to separately update some pieces of information while continuing to maintain some existing information. The role of the BG in this system builds on the more established role of the BG in motor control, by interacting in very similar circuits with PFC instead of motor cortex. In both cases, the BG provide a gating signal for determining whether or not a given frontal cortical 'action' should be executed or not. It's just that PFC actions are more cognitive than motor cortex, and include things like updating of working memory states, or of goals, plans, etc. Once updated, these PFC representations can then provide that top-down cognitive control mentioned above, and hence can shape action selection in BG-motor circuits, but also influence attention to task-relevant features in sensory cortex. Interestingly, the mechanisms for reinforcing which cognitive actions to execute (including whether or not to update working memory, or to attend to particular features, or to initiate a high level plan) seem to depend on very similar dopaminergic reinforcement learning mechanisms that are so central to motor control. This framework also provides a link between motivation and cognition which is very similar to the well established link between motivation and action. 129 | 130 | ## Language: All Together Now 131 | 132 | Language taps the coordinated function of many of the brain areas discussed above. Language requires highly sophisticated perceptual abilities, to be able to discriminate different speech sounds and different letters and combinations thereof (just listen and look at an unfamiliar foreign language to experience how amazing your own native language perceptual abilities are, which you take for granted). Likewise, sophisticated motor output abilities are required to produce the sounds and write the letters and words of language. In between, language requires some of the most demanding forms of cognitive processing, to keep track of the grammatical and semantic information streaming past you, often at high speed. This requires sophisticated executive function and working memory abilities, in addition to powerful distributed posterior-cortical semantic representations to integrate all the semantic information. 133 | 134 | Our exploration in the *Language* Chapter starts with considering how our brains represent the meaning of words. We explore how a self-organizing network can learn to encode statistical regularities in **word co-occurance**, that give rise to **semantic** representations that are remarkably effective in capturing the similarity structure of words [@LandauerDumais97]. We train this network on an early draft of the first edition of this text, so you should be familiar with the relevant semantics! 135 | 136 | We then turn to a model of the pathway from **orthography** (writing) to **phonology** (speech), to explore issues with **regularities and exceptions** in this spelling-to-sound mapping, which has been the topic of considerable debate [@SeidenbergMcClelland89; @PinkerPrince88; @PlautMcClellandSeidenbergEtAl96]. We show that the object recognition model from the Perception chapter has an important blend of features that support both regular and exception mappings, and this model pronounces nonword probe inputs much like people do, demonstrating that it has extracted similar underlying knowledge about the English mapping structure. 137 | 138 | We next bring the components of these first two models together in a small-scale model of reading, which interconnects orthographic, phonological, and semantic representations of individual words. These components form a **distributed lexicon.** There isn't one place where all word information is stored; instead it is distributed across brain areas that are specialized for processing the relevant perceptual, motor, and semantic information. Interestingly, we can simulate various forms of **acquired dyslexia** (e.g., from stroke or other forms of brain injury) by damaging specific pathways in this model, providing an important way of establishing neural correlates of language function in humans, where invasive experiments are not possible [@PlautShallice93]. 139 | 140 | Finally, we tackle the interactions between syntax and semantics in the context of processing the meaning of sentences, using the notion of a **sentence gestalt** representation, that uses coarse-coded distributed representations to encode the overall meaning of the sentence, integrating both syntactic and semantic cues [@StJohnMcClelland90]. This is a distinctly neural approach to syntax, as contrasted with the symbolic, highly structured approaches often employed in linguistic theories. 141 | -------------------------------------------------------------------------------- /chapter-06.md: -------------------------------------------------------------------------------- 1 | --- 2 | bibfile: ccnlab.bib 3 | --- 4 | 5 | # Perception and Attention {#sec:ch-percept} 6 | 7 | Perception is at once obvious and mysterious. It is so effortless to us that we have little appreciation for all the amazing computation that goes on under the hood. And yet we often use terms like "vision" as a metaphor for higher-level concepts (does the President have a vision or not?) --- perhaps this actually reflects a deep truth: that much of our higher-level cognitive abilities depend upon our perceptual processing systems for doing a lot of the hard work. Perception is not the mere act of seeing, but is leveraged whenever we imagine new ideas, solutions to hard problems, etc. Many of our most innovative scientists (e.g., Einstein, Richard Feynman) used visual reasoning processes to come up with their greatest insights. Einstein tried to visualize catching up to a speeding ray of light (in addition to trains stretching and contracting in interesting ways), and one of Feynman's major contributions was a means of visually diagramming complex mathematical operations in quantum physics. 8 | 9 | Pedagogically, perception serves as the foundation for our entry into cognitive phenomena. It is the most well-studied and biologically grounded of the cognitive domains. As a result, we will cover only a small fraction of the many fascinating phenomena of perception, focusing mostly on vision. But we do focus on a core set of issues that capture many of the general principles behind other perceptual phenomena. 10 | 11 | We begin with a computational model of **primary visual cortex (V1)**, which shows how self-organizing learning principles can explain the origin of oriented edge detectors, which capture the dominant statistical regularities present in natural images. This model also shows how excitatory lateral connections can result in the development of **topography** in V1 --- neighboring neurons tend to encode similar features, because they have a tendency to activate each other, and learning is determined by activity. 12 | 13 | Building on the features learned in V1, we explore how higher levels of the **ventral what** pathway can learn to **recognize objects** regardless of considerable variability in the superficial appearance of these objects as they project onto the retina. Object recognition is the paradigmatic example of how a hierarchically-organized sequence of feature category detectors can incrementally solve a very difficult overall problem. Computational models based on this principle can exhibit high levels of object recognition performance on realistic visual images, and thus provide a compelling suggestion that this is likely how the brain solves this problem as well. 14 | 15 | Next, we consider the role of the **dorsal where** (or how) pathway in **spatial attention**. Spatial attention is important for many things, including object recognition when there are multiple objects in view --- it helps focus processing on one of the objects, while degrading the activity of features associated with the other objects, reducing potential confusion. Our computational model of this interaction between what and where processing streams can account for the effects of brain damage to the *where* pathway, giving rise to hemispatial neglect for damage to only one side of the brain, and a phenomenon called Balint's syndrome with bilateral damage. This ability to account for both neurologically intact and brain damaged behavior is a powerful advantage of using neurally-based models. 16 | 17 | As usual, we begin with a review of the biological systems involved in perception. 18 | 19 | ## Biology of Perception 20 | 21 | Our goal in this section is to understand just enough about the biology to get an overall sense of how information flows through the visual system, and the basic facts about how different parts of the system operate. This will serve to situate the models that come later, which provide a much more complete picture of each step of information processing. 22 | 23 | ![The pathway of early visual processing from the retina through lateral geniculate nucleus of the thalamus (LGN) to primary visual cortex (V1), showing how information from the different visual fields (left vs. right) are routed to the opposite hemisphere.](figures/fig_vis_optic_path.png){#fig:fig-vis-optic-path width=40% } 24 | 25 | ![How the retina compresses information by only responding to areas of contrasting illumination, not solid uniform illumination. The response properties of retinal cells can be summarized by these Difference-of-Gaussian (DoG) filters, with a narrow central region and a wider surround (also called center-surround receptive fields). The excitatory and inhibitory components exactly cancel when both are uniformly illuminated, but when light falls more on the center vs. the surround (or vice-versa), they respond, as illustrated with an edge where illumination transitions between darker and lighter.](figures/fig_on_off_rfields_edge.png){#fig:fig-on-off-rfields-edge width=50% } 26 | 27 | ![A V1 simple cell that detects an oriented edge of contrast in the image, by receiving from a line of LGN on-center cells aligned along the edge. The LGN cells will fire due to the differential excitation vs. inhibition they receive (see previous figure), and then will activate the V1 neuron that receives from them.](figures/fig_v1_simple_dog_edge.png){#fig:fig-v1-simple-dog-edge width=30% } 28 | 29 | [@fig:fig-vis-optic-path] shows the basic optics and transmission pathways of visual signals, which come in through the retina, and progress to the lateral geniculate nucleus of the thalamus (LGN), and then to primary visual cortex (V1). The primary organizing principles at work here, and in other perceptual modalities and perceptual areas more generally, are: 30 | 31 | * **Transduction of different information** --- in the retina, photoreceptors are sensitive to different **wavelengths of light** (red = long wavelengths, green = medium wavelengths, and blue = short wavelengths), giving us color vision, but the retinal signals also differ in their **spatial frequency** (how coarse or fine of a feature they detect --- photoreceptors in the central *fovea* region can have high spatial frequency = fine resolution, while those in the periphery are lower resolution), and in their **temporal response** (fast vs. slow responding, including differential sensitivity to motion). 32 | 33 | * **Organization of information in a topographic fashion** --- for example, the left vs. right visual fields are organized into the contralateral hemispheres of cortex --- as the figure shows, signals from the left part of visual space are routed to the right hemisphere, and vice-versa. Information within LGN and V1 is also organized topographically in various ways. This organization generally allows similar information to be contrasted, producing an enhanced signal, and also grouped together to simplify processing at higher levels. 34 | 35 | * **Extracting relevant signals, while filtering irrelevant ones** --- [@fig:fig-on-off-rfields-edge] shows how retinal cells respond only to **contrast**, not uniform illumination, by using **center-surround** receptive fields (e.g., on-center, off-surround, or vice-versa). Only when one part of this receptive field gets different amounts of light compared to the others do these neurons respond. Typically this arises with edges of contrast, where illumination transitions between light and dark, as shown in the figure --- these transitions are the most informative aspects of an image, while regions of constant illumination can be safely ignored. [@fig:fig-v1-simple-dog-edge] shows how these center-surround signals (which are present in the LGN as well) can be integrated together in V1 simple cells to detect the orientation of these edges --- these edge detectors form the basic vocabulary for describing images in V1. More complex shapes can then be constructed from these basic line/edge elements. V1 also contains complex cells that build upon the simple cell responses ([@fig:fig-v1-visual-filters-hypercols-basic]), providing a somewhat richer basic vocabulary. 36 | 37 | The following videos show how we know what these receptive fields look like: 38 | 39 | * Classic Hubel & Wiesel V1 receptive field mapping using old school projector stimuli: [YouTube Video](https://www.youtube.com/watch?v=KE952yueVLA) 40 | 41 | * Newer reverse correlation V1 receptive field mapping: [YouTube Video](https://www.youtube.com/watch?v=n31XBMSSSpI) 42 | 43 | ![Complex and simple cell types within V1: The complex cells (top half of figure) integrate over the simple cell properties, including: creating larger receptive fields by integrating over multiple locations (the V1-Simple-Max cells do only this spatial integration) and abstracting across the polarity (positions of the on vs. off coding regions). The End-Stop cells are the most complex, detecting any form of contrasting orientation adjacent to a given simple cell. The V1 simple cells (bottom half of figure) detect oriented edges of contrast, as elaborated in [@fig:fig-v1-simple-dog-edge]. In the simulator, the V1 simple cells are encoded more directly using Gabor filters, which mathematically describe their oriented edge sensitivity.](figures/fig_v1_visual_filters_hypercols_basic.png){#fig:fig-v1-visual-filters-hypercols-basic width=50% } 44 | 45 | In the auditory pathway, the cochlear membrane plays an analogous role to the retina, and it also has a topographic organization according to the frequency of sounds, producing the rough equivalent of a fourier transformation of sound into a spectrogram. This basic sound signal is then processed in auditory pathways to extract relevant patterns of sound over time, in much the same way as occurs in vision. 46 | 47 | ![Updated and simplified version of Felleman & Van Essen's (1991) diagram of the anatomical connectivity of visual processing pathways, starting with primary visual cortex (V1) and on up. The blue-shaded areas comprise the ventral *What* pathway, and green-shaded are the dorsal *Where*. Reproduced from Markov et al., (2014).](figures/fig_markov_et_al_cort_hier.png){#fig:fig-markov_cort_hier-6 width=50% } 48 | 49 | Moving up beyond the primary visual cortex, the perceptual system provides an excellent example of the power of hierarchically organized layers of neural detectors, as we discussed in the *Networks* Chapter. [@fig:fig-markov_cort_hier-6] shows the anatomical connectivity patterns of all of the major visual areas, from V1 and on up [@MarkovVezoliChameauEtAl14; @FellemanVanEssen91]. The specific patterns of connectivity allow a hierarchical structure to be extracted, as shown, even though there are many interconnections outside of a strict hierarchy as well. 50 | 51 | ![Division of *What* vs *Where* (ventral vs. dorsal) pathways in visual processing, based on the classic Ungerlieder and Mishkin (1982) framework.](figures/fig_vis_system_bio.png){#fig:fig-vis_system_bio-6 width=50% } 52 | 53 | [@fig:fig-vis_system_bio-6] puts these areas into their anatomical locations, showing more clearly a **what vs where** (**ventral vs dorsal**) split in visual processing [@UngerleiderMishkin82]. The projections going in a ventral direction from V1 to V4 to areas of **inferotemporal cortex (IT)** (TE, TEO, labeled as PIT for posterior IT in the previous figure) are important for recognizing the identity ("what") of objects in the visual input, while those going up through parietal cortex extract spatial ("where") information, including motion signals in area MT and MST. We will see later in this chapter how each of these visual streams of processing can function independently, and also interact together to solve important computational problems in perception. 54 | 55 | Here is a quick summary of the flow of information up the **what** side of the visual pathway (pictured on the left side of Figure 6.5): 56 | 57 | * **V1** --- primary visual cortex, which encodes the image in terms of oriented edge detectors that respond to edges (transitions in illumination) along different angles of orientation. We will see in the first simulation in this chapter how these edge detectors develop through self-organizing learning, driven by the reliable statistics of natural images. 58 | 59 | * **V2** --- secondary visual cortex, which encodes combinations of edge detectors to develop a vocabulary of intersections and junctions, along with many other basic visual features (e.g., 3D depth selectivity, basic textures, etc), that provide the foundation for detecting more complex shapes. These V2 neurons also encode these features in a broader range of locations, starting a process that ends up with IT neurons being able to recognize an object regardless of where it appears in the visual field (i.e., *invariant* object recognition). 60 | 61 | * **V4** --- detects more complex shape features, over an even larger range of locations (and sizes, angles, etc). 62 | 63 | * **IT-posterior** (PIT) --- detects entire object shapes, over a wide range of locations, sizes, and angles. For example, there is an area near the fusiform gyrus on the bottom surface of the temporal lobe, called the **fusiform face area (FFA)**, that appears especially responsive to faces. As we saw in the *Networks* Chapter, however, objects are encoded in distributed representations over a broad range of areas in IT. 64 | 65 | * **IT-anterior** (AIT) --- this is where visual information becomes extremely abstract and **semantic** in nature --- it can encode all manner of important information about different people, places and things. 66 | 67 | In contrast, the **where** aspect of visual processing going up in a dorsal directly through the parietal cortex (areas MT, VIP, LIP, MST) contains areas that are important for processing motion, depth, and other spatial features. 68 | 69 | ## Oriented Edge Detectors in Primary Visual Cortex 70 | 71 | ![Orientation tuning of an individual V1 neuron in response to bar stimuli at different orientations --- this neuron shows a preference for vertically oriented stimuli.](figures/fig_v1_orientation_tuning_data.jpg){#fig:fig-v1-orientation-tuning-data width=50% } 72 | 73 | ![Topographic organization of oriented edge detectors in V1 --- neighboring regions of neurons have similar orientation tuning, as shown in this colorized map where different colors indicate orientation preference as shown in panel C. Panel B shows how a full 360 degree loop of orientations nucleate around a central point --- these are known as pinwheel structures.](figures/fig_v1_orientation_cols_data.jpg){#fig:fig-v1-orientation-cols-data width=30% } 74 | 75 | Neurons in the primary visual cortex (V1) detect the orientation of edges or bars of light within their receptive field (RF --- the region of the visual field that a given neuron receives input from). [@fig:fig-v1-orientation-tuning-data] shows characteristic data from electrophysiological recordings of an individual V1 neuron in response to oriented bars. This neuron responds maximally to the vertical orientation, with a graded fall off on either side of that. This is a very typical form of **tuning curve**. [@fig:fig-v1-orientation-cols-data] shows that these orientation tuned neurons are organized **topographically**, such that neighbors tend to encode similar orientations, and the orientation tuning varies fairly continuously over the surface of the cortex. 76 | 77 | The question we attempt to address in this section is why would such a topographical organization of oriented edge detectors exist in the primary visual cortex? There are multiple levels of answer to this question. At the most abstract level, these oriented edges are the basic constituents of the kinds of images that typically fall on our retinas. These are the most obvious **statistical regularities** of natural images [@OlshausenField96]. If this is indeed the case, then we would expect that the self-organizing aspect of the XCAL learning algorithm used in our models (as discussed in the *Learning* Chapter) would naturally extract these statistical regularities, providing another level of explanation: V1 represents oriented edge detectors because this is what the learning mechanisms will naturally develop. 78 | 79 | The situation here is essentially equivalent to the self organizing learning model explored in the *Learning* Chapter, which was exposed to horizontal and vertical lines, and learned to represent these strong statistical regularities in the environment. 80 | 81 | However, that earlier simulation did nothing to address the topography of the V1 neurons --- why do neighbors tend to encode similar information? The answer we explore in the following simulation is that neighborhood-level connectivity can cause nearby neurons to tend to activate together, and because activity drives learning, this then causes them to tend to learn similar things. 82 | 83 | ### Simulation Exploration 84 | 85 | ![Topographic organization of oriented edge detectors in simulation of V1 neurons exposed to small windows of natural images (mountains, trees, etc). The neighborhood connectivity of neurons causes a topographic organization to develop.](figures/fig_v1rf_map.png){#fig:fig-v1rf-map width=50% } 86 | 87 | Open `v1rf` in [CCN Sims](https://compcogneuro.org/simulations) to explore the development of oriented edge detectors in V1. This model gets exposed to a set of natural images, and learns to encode oriented edges, because they are the statistical regularity present in these images. [@fig:fig-v1rf-map] shows the resulting map of orientations that develops. 88 | 89 | ## Invariant Object Recognition in the *What* Pathway 90 | 91 | Object recognition is the defining function of the ventral "what" pathway of visual processing: identifying what you are looking at. Neurons in the inferotemporal (IT) cortex can detect whole objects, such as faces, cars, etc, over a large region of visual space. This **spatial invariance** (where the neural response remains the same or invariant over spatial locations) is critical for effective behavior in the world --- objects can show up in all different locations, and we need to recognize them regardless of where they appear. Achieving this outcome is a very challenging process, one which has stumped artificial intelligence (AI) researchers for a long time --- in the early days of AI, the 1960's, it was optimistically thought that object recognition could be solved as a summer research project, and 50 years later we are making a lot of progress, but it remains unsolved in the sense that people are still much better than our models. Despite considerable progress in recent years, and claims that current models actually exceed human object recognition abilities, it remains the case that human visual perception does a much better job of actually seeing the underlying 3D structure of the world, whereas current AI models typically just do pattern discrimination at the level of a more texture-based encoding. This makes them susceptible to categorization errors that people would never make. In any case, because our brains do object recognition effortlessly all the time, we do not really appreciate how hard of a problem it is. 92 | 93 | ![Why object recognition is hard: things that should be categorized as the same (i.e., have the same output label) often have no overlap in their retinal input features when they show up in different locations, sizes, etc, but things that should be categorized as different often have high levels of overlap when they show up in the same location. Thus, the bottom-up similarity structure is directly opposed to the desired output similarity structure, making the problem very difficult.](figures/fig_objrec_difficulty.png){#fig:fig-objrec-difficulty width=40% } 94 | 95 | The reason object recognition is so hard is that there can often be no overlap at all among visual inputs of the same object in different locations (sizes, rotations, colors, etc), while there can be high levels of overlap among different objects in the same location ([@fig:fig-objrec-difficulty]). Therefore, you cannot rely on the bottom-up visual similarity structure --- instead it often works directly against the desired output categorization of these stimuli. As we saw in the *Learning* Chapter, successful learning in this situation requires error-driven learning, because self-organizing learning tends to be strongly driven by the input similarity structure. 96 | 97 | ![Schematic for how multiple levels of processing can result in invariant object recognition, where an object can be recognized at any location across the input. Each level of processing incrementally increases the featural complexity and spatial invariance of what it detects. Doing this incrementally allows the system to appropriately bind together features and their relationships, while also gradually building up overall spatial invariance.](figures/fig_invar_trans.png){#fig:fig-invar-trans width=50% } 98 | 99 | ![Another way of representing the hierarchy of increasing featural complexity that arises over the areas of the ventral visual pathways. V1 has elementary feature detectors (oriented edges). Next, these are combined into junctions of lines in V2, followed by more complex visual features in V4. Individual faces are recognized at the next level in IT (even here multiple face units are active in graded proportion to how similar people look). Finally, at the highest level are important functional "semantic" categories that serve as a good basis for actions that one might take --- being able to develop such high level categories is critical for intelligent behavior --- this level corresponds to more anterior areas of IT.](figures/fig_category_hierarch_dist_reps.png){#fig:fig-category-hierarch-dist-reps-6 width=100% } 100 | 101 | The most successful approach to the object recognition problem, which was advocated initially in a model by [@Fukushima80], is to incrementally solve two problems over a hierarchically organized sequence of layers [@fig:fig-invar-trans; @fig:fig-category-hierarch-dist-reps-6]: 102 | 103 | * The invariance problem, by having each layer *integrate over a range of locations* (and sizes, rotations, etc) for the features in the previous layer, such that neurons become increasingly invariant as one moves up the hierarchy. 104 | 105 | * The pattern discrimination problem (distinguishing an A from an F, for example), by having each layer build up more complex combinations of feature detectors, as a result of detecting *combinations of the features* present in the previous layer, such that neurons are better able to discriminate even similar input patterns as one moves up the hierarchy. 106 | 107 | The critical insight from these models is that breaking these two problems down into incremental, hierarchical steps enables the system to solve both problems without one causing trouble for the other. For example, if you had a simple fully invariant vertical line detector that responded to a vertical line in any location, it would be impossible to know what spatial relationship this line has with other input features, and this relationship information is critical for distinguishing different objects (e.g., a T and L differ only in the relationship of the two line elements). So you cannot solve the invariance problem in one initial pass, and then try to solve the pattern discrimination problem on top of that. They must be interleaved, in an incremental fashion. Similarly, it would be completely impractical to attempt to recognize highly complex object patterns at each possible location in the visual input, and then just do spatial invariance integration over locations after that. There are way too many different objects to discriminate, and you'd have to learn about them anew in each different visual location. It is much more practical to incrementally build up a "part library" of visual features that are increasingly invariant, so that you can learn about complex objects only toward the top of the hierarchy, in a way that is already spatially invariant and thus only needs to be learned once. 108 | 109 | ![Summary of neural response properties in V2, V4, and IT for macaque monkeys. The Smax / MAX column shows how the response to simple visual inputs (Smax) compares to the maximum response to any visual input image tested (MAX). The other column shows the overall size of the visual receptive field, over which the neurons exhibit relatively invariant responses to visual features. For V2, nearly all neurons responded maximally to simple stimuli, and the receptive field sizes were the smallest. For V4, only 50% of neurons had simple responses as their maximal response, and the receptive field sizes increase over V2. Posterior IT increases (slightly) on both dimensions, while anterior IT exhibits almost entirely complex featural responding and significantly larger receptive fields. These incremental increases in complexity and invariance (receptive field size) are exactly as predicted by the incremental computational solution to invariant object recognition as shown in the previous figure. Reproduced from Kobatake & Tanaka (1994).](figures/fig_kobatake_tanaka_94_invariance.png){#fig:fig-kobatake-tanaka-94-invariance width=50% } 110 | 111 | ![Complex stimuli that evoked a maximal response from neurons in V2, V4, and IT, providing some suggestion for what kinds of complex features these neurons can detect. Most V2 neurons responded maximally to simple stimuli (oriented edges, not shown). Reproduced from Kobatake & Tanaka (1994).](figures/fig_kobatake_tanaka_94_v2_v4_it.png){#fig:fig-kobatake-tanaka-94-v2-v4-it width=50% } 112 | 113 | In a satisfying convergence of top-down computational motivation and bottom-up neuroscience data, this incremental, hierarchical solution provides a nice fit to the known properties of the visual areas along the ventral what pathway (V1, V2, V4, IT). [@fig:fig-kobatake-tanaka-94-invariance] summarizes neural recordings from these areas in macaque monkeys [@KobatakeTanaka94], and shows that as one proceeds up the hierarchy of areas, neurons increase in the complexity of the stimuli that drive their responding, and the size of the receptive field over which they exhibit an invariant response to these stimuli. [@fig:fig-kobatake-tanaka-94-v2-v4-it] shows example complex stimuli that evoked maximal responding in each of these areas, to give a sense of what kind of complex feature conjunctions these neurons can detect. 114 | 115 | ### Exploration of Object Recognition 116 | 117 | ![Set of 20 objects composed from horizontal and vertical line elements used for the object recognition simulation. By using a restricted set of visual feature elements, we can more easily understand how the model works, and also test for generalization to novel objects (object 18 and 19 are not trained initially, and then subsequently trained only in a relatively few locations --- learning there generalizes well to other locations).](figures/fig_objrec_objs.png){#fig:fig-objrec-objs width=30% } 118 | 119 | Open up the `objrec` simulation in [CCN Sims](https://compcogneuro.org/simulations) for the computational model of object recognition, which demonstrates the incremental hierarchical solution to the object recognition problem. We use a simplified set of "objects" ([@fig:fig-objrec-objs]) composed from vertical and horizontal line elements. This simplified set of visual features allows us to better understand how the model works, and also enables testing generalization to novel objects composed from these same sets of features. You will see that the model learns simpler combinations of line elements in area V4, and more complex combinations of features in IT, which are also invariant over the full receptive field. These IT representations are not identical to entire objects --- instead they represent an invariant distributed code for objects in terms of their constituent features. The generalization test shows how this distributed code can support rapid learning of new objects, as long as they share this set of features. Although they are likely much more complex and less well defined, it seems that a similar such vocabulary of visual shape features are learned in primate IT representations. 120 | 121 | ## Spatial Attention and Neglect in the *Where/How* Pathway 122 | 123 | The dorsal visual pathway that goes into the parietal cortex is more heterogeneous in its functionality relative to the object recognition processing taking place in the ventral *what* pathway, which appears to be the primary function of that pathway. Originally, the dorsal pathway was described as a *where* pathway, in contrast to the ventral *what* pathway [@UngerleiderMishkin82]. However, [@GoodaleMilner92] provide a compelling broader interpretation of this pathway as performing a *how* function --- mapping from perception to action. One aspect of this *how* functionality involves spatial location information, in that this information is highly relevant for controlling motor actions in 3D space, but spatial information is too narrow of a definition for the wide range of functions supported by the parietal lobe. Parietal areas are important for numerical and mathematical processing, and representation of abstract relationship information, for example. Areas of the parietal cortex also appear to be important for modulating episodic memory function in the medial temporal lobe, and various other functions. This later example may represent a broader set of functions associated with prefrontal cortical cognitive control --- areas of the parietal cortex are almost always active in combination with the prefrontal cortex in demanding cognitive control tasks, although there is typically little understanding of what precise role they might be playing. 124 | 125 | ![Where's Waldo visual search example.](figures/fig_wheres_waldo.jpg){#fig:fig-wheres-waldo width=50% } 126 | 127 | In this chapter, we focus on the established *where* aspect of parietal function, and we'll take up some of the *how* functionality in the next chapter on Motor Control. Even within the domain of spatial processing, there are many cognitive functions that can be performed using parietal spatial representations, but we focus here on their role in focusing attention to spatial locations. In relation to the previous section, one crucial function of spatial attention is to enable object recognition to function in visual scenes that have multiple different objects present. For example, consider one of those "where's Waldo" puzzles ([@fig:fig-wheres-waldo]) that is packed with rich visual detail. Is it possible to perceive such a scene all in one take? No. You have to scan over the image using a "spotlight" of visual attention to focus on small areas of the image, which can then be processed effectively by the object recognition pathway. The ability to direct this spotlight of attention depends on spatial representations in the dorsal pathway, which then interact with lower levels of the object recognition pathway (V1, V2, V4) to constrain the inputs to reflect only those visual features that come from within this spotlight of attention. 128 | 129 | ### Hemispatial Neglect 130 | 131 | ![Drawings of given target objects by patients with hemispatial neglect, showing profound neglect of the left side of the drawings.](figures/fig_neglect_drawings.png){#fig:fig-neglect-drawings width=30% } 132 | 133 | ![Progression of self portraits by an artist with hemispatial neglect, showing gradual remediation of the neglect over time.](figures/fig_neglect_portrait.jpg){#fig:fig-neglect-portrait width=30% } 134 | 135 | ![Results of a line bisection task for a person with hemispatial neglect. Notice that neglect appears to operate at two different spatial scales here: for the entire set of lines, and within each individual line.](figures/fig_neglect_line_bisect.png){#fig:fig-neglect-line-bisect width=50% } 136 | 137 | Some of the most striking evidence that the parietal cortex is important for spatial attention comes from patients with hemispatial neglect, who tend to ignore or neglect one side of space (Figures 6.17, 6.18, 6.19). This condition typically arises from a stroke or other form of brain injury affecting the right parietal cortex, which then gives rise to a neglect of the left half of space (due to the crossing over of visual information shown in the biology section). Interestingly, the neglect applies to multiple different spatial reference frames, as shown in [@fig:fig-neglect-line-bisect], where lines on the left side of the image tend to be neglected, and also each individual line is bisected more toward the right, indicating a neglect of the left portion of each line. 138 | 139 | ### The Posner Spatial Cueing Task 140 | 141 | ![The Posner spatial cueing task, widely used to explore spatial attention effects. The participant is shown a display with two boxes and a central fixation cross --- on some trials, one of the boxes is cued (e.g., the lines get transiently thicker), and then a target appears in one of the boxes (or not at all on catch trials). The participant just presses a key when they first detect the target. Reaction time is quicker for valid cues vs. invalid ones, suggesting that spatial attention was drawn to that side of space. Patients with hemispatial neglect exhibit slowing for targets that appear in the neglected side of space, particularly when invalidly cued.](figures/fig_posner_task.png){#fig:fig-posner-task width=50% } 142 | 143 | One of the most widely used tasks to study the spotlight of spatial attention is the Posner spatial cueing task, developed by Michael Posner [@Posner80] ([@fig:fig-posner-task]). One side of visual space is cued, and the effects of this cue on subsequent target detection are measured. If the cue and target show up in the same side of space (*valid* cue condition), then reaction times are faster compared to when they show up on different sides of space (*invalid* cue condition). This difference in reaction time (RT) suggests that spatial attention is drawn to the cued side of space, and thus facilitates target detection. The invalid case is actually worse than a neutral condition with no cue at all, indicating that the process of reallocating spatial attention to the correct side of space takes some amount of time. Interestingly, this task is typically run with the time interval between cue and target sufficiently brief as to prevent eye movements to the cue --- thus, these attentional effects are described as *covert attention*. 144 | 145 | ![Typical data from the Posner spatial cueing task, showing a speedup for valid trials, and a slowdown for invalid trials, compared to a neutral trial with no cueing. The data for patients with hemispatial neglect is also shown, with their overall slower reaction times normalized to that of the intact case.](figures/fig_posner_graph.png){#fig:fig-posner-graph width=30% } 146 | 147 | As shown in [@fig:fig-posner-graph], patients with hemispatial neglect show a disproportionate increase in reaction times for the invalid cue case, specifically when the cue is presented to the good visual field (typically the right), while the target appears in the left. Posner took this data to suggest that these patients have difficulty disengaging attention, according to his box-and-arrow model of the spatial cueing task ([@fig:fig-posner-disengage-mech]). 148 | 149 | ![Posner's box-and-arrow model of the spatial cueing task. He suggests that spatial neglect deficits are attributable to damage to the disengage mechanism. However, this fails to account for the effects of bilateral parietal damage, for example.](figures/fig_posner_disengage_mech.png){#fig:fig-posner-disengage-mech width=20% } 150 | 151 | ![Interacting spatial and object recognition pathways can explain Posner spatial attention effects in terms of spatial influences on early object recognition processing, in addition to top-down influence on V1 representations.](figures/fig_spat_attn_lateral.png){#fig:fig-spat-attn-lateral width=30% } 152 | 153 | We explore an alternative account here, based on bidirectional interactions between spatial and object processing pathways ([@fig:fig-spat-attn-lateral]). In this account, damage to one half of the spatial processing pathway leads to an inability of that side to compete against the intact side of the network. Thus, when there is something to compete against (e.g., the cue in the cueing paradigm), the effects of the damage are most pronounced. 154 | 155 | Importantly, these models make distinct predictions regarding the effects of *bilateral* parietal damage. Patients with this condition are known to suffer from Balint's syndrome, which is characterized by a profound inability to recognize objects when more than one is present in the visual field [@CoslettSaffran91]. This is suggestive of the important role that spatial attention plays in facilitating object recognition in crowded visual scenes. According to Posner's disengage model, bilateral damage should result in difficulty disengaging from *both* sides of space, producing slowing in invalid trials for both sides of space. In contrast, the competition-based model makes the opposite prediction: the lesions serve to reduce competition on *both* sides of space, such that there should be reduced attentional effects on both sides. That is, the effect of the invalid cue actually decreases in magnitude. The data are consistent with the competition model, and not Posner's model [@VerfaellieRapcsakHeilman90]. 156 | 157 | ### Exploration of Spatial Attention 158 | 159 | Open `attn_simpl` in [CCN Sims](https://compcogneuro.org/simulations) to explore a model with spatial and object pathways interacting in the context of multiple spatial attention tasks, including perceiving multiple objects, and the Posner spatial cueing task. It reproduces the behavioral data shown above, and correctly demonstrates the observed pattern of reduced attentional effects for Balint's patients. 160 | 161 | 162 | -------------------------------------------------------------------------------- /chapter-10.md: -------------------------------------------------------------------------------- 1 | --- 2 | bibfile: ccnlab.bib 3 | --- 4 | 5 | # Language {#sec:ch-lang} 6 | 7 | Language involves almost every part of the brain, as covered in other chapters in the text: 8 | 9 | * *Perception and attention:* language requires the perception of words from auditory sound waves, and written text. Attention is critical for pulling out individual words on the page, and individual speakers in a crowded room. In this chapter, we see how a version of the object recognition model from the perception chapter can perform written word recognition, in a way that specifically leverages the spatial invariance property of this model. 10 | 11 | * *Motor control*: Language production obviously requires motor output in the form of speech, writing, etc. Fluent speech depends on an intact cerebellum, and the basal ganglia have been implicated in a number of linguistic phenomena. 12 | 13 | * *Learning and memory:* early word learning likely depends on episodic memory in the hippocampus, while longer-term memory for word meaning depends on slow integrated learning in the cortex. Memory for recent topics of discourse and reading (which can span months in the case of reading a novel) likely involves the hippocampus and sophisticated semantic representations in temporal cortex. 14 | 15 | * *Executive Function:* language is a complex mental facility that depends critically on coordination and working memory from the prefrontal cortex (PFC) and basal ganglia --- for example encoding syntactic structures over time, pronoun binding, and other more transient forms of memory. 16 | 17 | One could conclude from this that language is not particularly special, and instead represents a natural specialization of *domain general* cognitive mechanisms. Of course, people have specialized articulatory apparatus for producing speech sounds, which are not shared by other primate species, but one could argue that everything on top of this is just language infecting pre-existing cognitive brain structures. Certainly reading and writing is too recent to have any evolutionary adaptations to support it (but it is also the least "natural" aspect of language, requiring explicit schooling, compared to the essentially automatic manner in which people absorb spoken language). 18 | 19 | But language is fundamentally different from any other cognitive activity in a number of important ways: 20 | 21 | * **Symbols** --- language requires thought to be reduced to a sequence of symbols, transported across space and time, to be reconstructed in the receiver's brain. 22 | 23 | * **Syntax** --- language obeys complex abstract regularities in the ordering of words and letters/phonemes. 24 | 25 | * **Temporal extent and complexity** --- language can unfold over a very long time frame (e.g., Tolstoy's *War and Peace*), with a level of complexity and richness conveyed that far exceeds any naturally occurring experiences that might arise outside of the linguistic environment. If you ever find yourself watching a movie on an airplane without the sound, you'll appreciate that visual imagery represents the lesser half of most movie's content (the interesting ones anyway). 26 | 27 | * **Generativity** --- language is "infinite" in the sense that the number of different possible sentences that could be constructed is so large as to be effectively infinite. Language is routinely used to express new ideas. You may find some of those here. 28 | 29 | * **Culture** --- much of our intelligence is imparted through cultural transmission, conveyed through language. Thus, language shapes cognition in the brain in profound ways. 30 | 31 | The "special" nature of language, and its dependence on domain-general mechanisms, represent two poles in the continuum of approaches taken by different researchers. Within this broad span, there is plenty of room for controversy and contradictory opinions. Noam Chomsky famously and influentially theorized that we are all born with an innate universal grammar, with language learning amounting to discovering the specific parameters of that language instance. On the other extreme, connectionist language modelers such as Jay McClelland argue that completely unstructured, generic neural mechanisms (e.g., backpropagation networks) are sufficient for explaining (at least some of) the special things about language. 32 | 33 | Our overall approach is clearly based in the domain-general approach, given that the same general-purpose neural mechanisms used to explore a wide range of other cognitive phenomena are brought to bear on language here. However, we also think that certain features of the PFC / basal ganglia system play a special role in symbolic, syntactic processing. At present, these special contributions are only briefly touched upon here, and were elaborated a bit more in the executive function chapter, but future plans call for further elaboration. One hint at these special contributions comes from *mirror neurons* discovered in the frontal cortex of monkeys, in an area thought to be analogous to Broca's area in humans --- these neurons appear to encode the intentions of actions performed by other people (or monkeys), and thus may constitute a critical capacity to understand what other people are trying to communicate. 34 | 35 | We start as usual with a biological grounding to language, in terms of particularly important brain areas and the biology of speech. Then we consider the nature of semantic knowledge. We see how a self-organizing model (like the V1 receptive field model in the *Perception and Attention* Chapter) can encode word meaning in terms of the statistics of word co-occurrence, as developed in the Latent Semantic Analysis (LSA) model. We explore a large-scale reading model based on our object recognition model from the *Perception and Attention* Chapter. This model maps written words (orthography) to spoken motor output of words (phonology) and is capable of pronouncing the roughly 3,000 monosyllabic words in English and generalizing this knowledge to nonwords. Next, we explore a model that brings together semantics, orthography, and phonology and demonstrates how damage to different areas can give rise to distinctive patterns of acquired dyslexia. Finally, we explore syntax at the level of sentences in the Sentence Gestalt model, where syntactic and semantic information are integrated over time to form a "gestalt" like understanding of sentence meaning. 36 | 37 | ## Biology of Language 38 | 39 | ![Brain areas associated with two of the most well-known forms of aphasia, or deficit in speech produced by damage to these areas. Broca's aphasia is associated with impaired syntax but intact semantics, while Wernicke's is the opposite. This makes sense given their respective locations in brain: temporal cortex for semantics, and frontal cortex for syntax.](figures/fig_aphasia_areas.png){#fig:fig-aphasia-areas width=40% } 40 | 41 | The classic "textbook" brain areas for language are **Broca's** and **Wernicke's** areas ([@fig:fig-aphasia-areas]), which have been associated with syntax and semantics, respectively. For example, a person who suffers a stroke or other form of damage to Wernicke's area can produce fluent, syntactically-correct speech, which is essentially devoid of meaning. Here is one example: 42 | 43 | > *"You know that smoodle pinkered and that I want to get him round and take care of him like you want before"* 44 | 45 | which apparently was intended to mean: "The dog needs to go out so I will take him for a walk." 46 | 47 | In contrast, a person with damage to Broca's area has difficulty producing syntactically correct speech output, typically producing single content words with some effort, e.g., "dog....walk". 48 | 49 | The more modern term for Broca's aphasia is *expressive aphasia*, indicating a primary deficit in expressing speech. Comprehension is typically intact, although interestingly there can be deficits in understanding more syntactically complex sentences. Wernicke's aphasia is known as *receptive aphasia*, indicating a deficit in comprehension, but also expression of meaning. 50 | 51 | Biologically, the locations of the damage associated with these aphasias are consistent with what we know about these areas more generally. The ventral posterior area of frontal cortex known as Broca's area (corresponding to Brodmann's areas 44 and 45) is adjacent to the primary motor area associated with control over the mouth, and thus it represents supplementary motor cortex for vocal output. Even though Broca's patient's can physically move their mouths and other articulatory systems, they cannot perform the complex sequencing of these motor commands that is necessary to produce fluid speech. Interestingly, these higher-order motor control areas also seem to be important for syntactic processing, even for comprehension. This is consistent with the idea that frontal cortex is important for temporally-extended patterning of behavior according to increasingly complex plans as one moves more anterior in frontal cortex. 52 | 53 | The location of Wernicke's area in temporal cortex is sensible, given that we know that the temporal lobe represents the semantic meanings of objects and other things. 54 | 55 | There are still some controversies about the exact nature of damage required to produce each of these aphasias (and likely a large amount of individual variability across people as well), but the basic distinction between these broad areas remains quite valid. 56 | 57 | ### The Articulatory Apparatus and Phonology 58 | 59 | ![The different components of the vocal tract, which are important for producing the range of speech sounds that people can produce.](figures/fig_vocal_tract.png){#fig:fig-vocal-tract width=40% } 60 | 61 | ![Left panel: International Phonetic Alphabet (IPA) for vowels, as a function of where the tongue is positioned (front vs. back, organized horizontally in figure), and the shape of the lips (vertical axis in figure) --- these two dimensions define a space of vowel sounds. Right panel: Version of IPA vowel space with vowel labels used by PMSP and in our simulations --- these are all standard roman letters and thus easier to manipulate in computer programs. Only the subset present in English is used. ](figures/fig_ipa_chart_vowels_ipa_pmsp.png){#fig:fig-ipa-chart-vowels-ipa-pmsp width=75% } 62 | 63 | ![International Phonetic Alphabet (IPA) for consonants, which are defined in terms of the location where the flow of air is restricted (place, organized horizontally in the table) and the manner in which it is restricted (plosive, fricative, etc, organized vertically).](figures/fig_ipa_chart_consonants_simple.png){#fig:fig-ipa-chart-consonants-simple width=100% } 64 | 65 | The vocal tract in people ([@fig:fig-vocal-tract]) is capable of producing a wide range of different speech sounds, by controlling the location and manner in which sound waves are blocked or allowed to pass. There are two basic categories of speech sounds: vowels and consonants. Vowels occur with unobstructed airflow (you can sing a vowel sound over an extended period), and differ in the location of the tongue and lips ([@fig:fig-ipa-chart-vowels-ipa-pmsp]). For example, the long "E" vowel sound as in "seen" is produced with the tongue forward and the lips relatively closed. Consonants involve the blockage of airflow, in a variety of locations, and with a variety of different manners ([@fig:fig-ipa-chart-consonants-simple]). The "s" consonant is a "fricative" (friction-like obstruction of the sound) with the tongue placed at the aveloar ridge. It is also unvoiced, which means that the vocal chords are not vibrating for it --- the "z" sound is just like an "s" except it is voiced. 66 | 67 | To see a video of the movements of the tongue in vocal output, see this [YouTube link](https://www.youtube.com/watch?v=M2OdAp7MJAI). 68 | 69 | We'll take advantage of these phonological features in the output of our detailed reading model --- using these features ensures that the spelling-to-sound correspondences actually capture the real phonological structure of the English language (at least at a fairly abstract level). A more detailed motor model of speech output developed by Frank Guenther, which we hope to include in our models at some point, can be found [here](http://sites.bu.edu/guentherlab/software/). 70 | 71 | ## Latent Semantics in Word Co-Occurrence 72 | 73 | ![Distributed semantics for concrete words, where different aspects of a word's meaning are encoded in domain-specific brain areas (e.g., the sound of thunder in auditory areas, and the feel of velvet in somatosensory areas). Figure adapted from Allport, 1985.](figures/fig_distrib_sem.png){#fig:fig-distrib-sem width=40% } 74 | 75 | The first language model we explore focuses on semantics. What is the nature of the semantic representations that encode the meaning of words? An increasing body of data supports the idea shown in [@fig:fig-distrib-sem], where the meaning of concrete words is encoded by patterns of activity within domain-specific brain areas that process sensory and motor information [@Allport85]. Thus, semantics is distributed throughout a wide swath of the brain, and it is fundamentally *embodied* and *grounded* in the sensory-motor primitives that we first acquire in life. Thus, the single "semantics" area shown later in the triangle model is a major simplification relative to the actual widely distributed nature of semantic meaning in the brain. 76 | 77 | However, there is also increasing evidence that the anterior tip or "pole" of the temporal lobe plays a particularly important role in representing semantic information, perhaps most importantly for more abstract words that lack a strong sensory or motor correlate. One theory is that this area acts as a central "hub" for coordinating the otherwise distributed semantic information [@PattersonNestorRogers07]. 78 | 79 | How do we learn the meanings of these more abstract words in the first place? Unlike the more concrete words shown in [@fig:fig-distrib-sem], the meanings of more abstract words cannot be so easily pushed off to sensory and motor areas. One compelling idea here is that words obtain their meaning in part from the company they keep --- the statistics of word co-occurrence across the large volume of verbal input that we are exposed to can actually provide clues as to what different words mean. One successful approach to capturing this idea in a functioning model is called *Latent Semantic Analysis* (LSA) [@LandauerDumais97] --- see [LSA Website](http://lsa.colorado.edu) for full details and access to this model. 80 | 81 | LSA works by recording the statistics of how often words co-occur with each other within semantically-relevant chunks of text, typically paragraphs. However, these surface statistics themselves are not sufficient, because for example synonyms of words occur together relatively rarely, compared to how closely related they should be. And in general, there is a lot of variability in word choice and idiosyncrasies of word choices that are reflected in these detailed statistics. The key step that LSA takes in dealing with this problem is to apply a *dimensionality reduction* technique called *Singular Value Decomposition* (SVD), which is closely related to *Principal Components Analysis* (PCA), which in turn is closely related to the Hebbian self-organizing learning that our neural network models perform. 82 | 83 | The key result of this SVD/PCA/Hebbian process is to extract the *strongest groupings* or clusters of words that co-occur together, in a way that integrates over many partially-overlapping subsets of word groups. Thus, even though synonyms do not tend to occur with each other, they do co-occur with many of the same other sets of words, and this whole group of words represents a strong statistical grouping that will be pulled out by the dimensionality reduction / Hebbian self-organizing learning process. 84 | 85 | This process is exactly the same as what we saw with the V1 receptive field model in the *Perception and Attention* Chapter. In that model, Hebbian learning extracted the statistical regularity of oriented edges from a set of natural images. Any given image typically contains a noisy, partial version of an oriented edge, with perhaps several pixels occluded or blurry or otherwise distorted. However, as the self-organizing learning process integrates over many such inputs, these idiosyncrasies wash away, and the strongest statistical groupings of features emerge as oriented edges. 86 | 87 | Unlike the V1 model, however, the individual statistical clusters that emerge from the LSA model (including our Hebbian version of it) do not have any clear interpretation equivalent to "oriented edges". As you'll see in the exploration, you can typically make some sense of small subsets of the words, but no obvious overall meaning elements are apparent. But this is not a problem --- what really matters is that the overall distributed pattern of activity across the semantic layer appropriately captures the meanings of words. And indeed this turns out to be the case. 88 | 89 | ### Exploration 90 | 91 | Run the `sem` model from [CCN Sims](https://compcogneuro.org/simulations) for the exploration of semantic learning of word co-occurrences. The model here was trained on an early draft of the first edition of this textbook, and thus has relatively specialized knowledge, hopefully much of which is now shared by you the reader. 92 | 93 | ## Spelling to Sound Mappings in Word Reading 94 | 95 | We now turn to the pathway between visual word inputs (orthography) and verbal speech output (phonology), using a large set of words comprising most of the monosyllabic words in English (nearly 3,000 words). By learning on such a large selection of words, sampled according to their frequency of occurrence in English, the network has a chance to extract the "rules" that govern the mapping between spelling and sound in English (such as they are), and to thus be able to successfully pronounce *nonwords*. 96 | 97 | English is a particularly difficult language from a pronunciation perspective, as anyone knows who has tried to acquire it as a second language. There are very few (if any) absolute rules. Everything is more of a *partial, context-dependent regularity*, which is also called a *subregularity*. For example, compare the pronunciation of the letter *i* in *mint* and *hint* (short *i* sound) to that in *mind* and *find* (long *I* sound). The final consonant (*t* vs. *d*) determines the pronunciation, and of course there are always exceptions such as *pint* (long *I* sound). 98 | 99 | One way to classify how strong a regularity is, is to count how many other letters the pronunciation depends upon. A complete exception like *pint* or *yacht* depends on *all* the letters in the word, while *mint* vs. *mind* depends on one other letter in the word (the final *t* or *d*). There are many silent letter examples, such as the final *e* in many words. A nice subregularity is the letter *m*, which depends on whether there is an *n* next to it, in which case it goes silent, as in *damn*, *column*, or *mnemonic*. Many other consonants can be silent with varying degrees of subregularity, including *b* (*debt*), *d* (*handsome*), *h* (*honest*), *l* (*halve*), *p* (*coup*), *r* (*iron* in British English), *s* (*aisle*), *t* (*castle*), *w* (*sword*), and *z* (*rendezvous*). 100 | 101 | Another factor that determines how much context is required to pronounce a given letter is the preponderance of multi-letter groups like *th* (*think*), which have a particular regular pronunciation that differs from the individual letters separately. Other examples of these include: *sch* (*school*), *tch* (*batch*), *gh* (*ghost*), *ght* (*right*), *kn* (*knock*), *ph* (*photo*), *wh* (*what*). One of the most context sensitive set of letters is the *ough* group, as in *though, tough, cough, plough, through, nought*, where the pronunciation varies widely. 102 | 103 | So English is a mess. The constructed word *ghoti* is a famous example of how crazy it can get. It is pronounced "fish", where the *gh* is an *f* sound as in *tough*, *o* is an *i* sound as in *women*, and *ti* is a *sh* sound as in *nation*. 104 | 105 | For any system to be able to have any chance of producing correct pronunciation of English, it must be capable of taking into account a range of context around a given letter in a word, all the way up to the entire word itself. An influential early approach to simulating spelling to sound in a neural network [@SeidenbergMcClelland89] used a so-called *Wickelfeature* representation (named after Wayne Wickelgren), where the written letters were encoded in groups of three. For example, the word "think" would be encoded as *thi*, *hin*, and *ink*. This is good for capturing context, but it is a bit rigid, and doesn't allow for the considerable amount of regularity in individual letters themselves (most of the time, an *m* is just an *m*). As a result, this model did not generalize very well to nonwords, where letters showed up in different company than in the real words used in training. A subsequent model by [@PlautMcClellandSeidenbergEtAl96] (hereafter *PMSP*) achieved good nonword generalization by representing input words through a hand-coded combination of individual letter units and useful multi-letter contexts (e.g., a *th* unit). 106 | 107 | ![Word reading as a process of spatially invariant object recognition. Words show up in different locations in the input, and the next level up, equivalent to the V4 level in the object recognition model, extracts more complex combinations of letters, while also developing more invariant representations that integrate individual letters or multi-letter features over multiple different locations. The IT level representation then has a fully spatially invariant representation of the word (as a distributed representation integrating over individual letters and letter groups), which then provides a nice mapping to the phonological output.](figures/fig_reading_model.png){#fig:fig-reading-model width=20% } 108 | 109 | We take a different approach in our spelling-to-sound model ([@fig:fig-reading-model]), leveraging ideas from the object recognition model that was explored in the *Perception and Attention* Chapter. Specifically, we saw that the object recognition model could learn to build up increasingly complex combinations of features, while also developing spatial invariance, over multiple levels of processing in the hierarchy from V1 through IT. In the context of word recognition, these complex features could include combinations of letters, while spatial invariance allows the system to recognize that an *m* in any location is the same as any other *m* (most of the time). 110 | 111 | One compelling demonstration of the importance of spatial invariance in reading comes from this example, which made the rounds in email a few years ago: 112 | 113 | > I cnduo't bvleiee taht I culod aulaclty uesdtannrd waht I was rdnaieg. Unisg the icndeblire pweor of the hmuan mnid, aocdcrnig to rseecrah at Cmabrigde Uinervtisy, it dseno't mttaer in waht oderr the lterets in a wrod are, the olny irpoamtnt tihng is taht the frsit and lsat ltteer be in the rhgit pclae. The rset can be a taotl mses and you can sitll raed it whoutit a pboerlm. Tihs is bucseae the huamn mnid deos not raed ervey ltteer by istlef, but the wrod as a wlohe. Aaznmig, huh? Yaeh and I awlyas tghhuot slelinpg was ipmorantt! See if yuor fdreins can raed tihs too. 114 | 115 | Clearly this is more effortful than properly spelled text, but the ability to read it at all indicates that just extracting individual letters in an invariant manner goes a long way. 116 | 117 | To test the performance of this object-recognition based approach, we ran it through a set of different standard sets of nonwords, several of which were also used to test the PMSP model. The results are shown in [@tbl:table-nonword]. 118 | 119 | * **Glushko regulars** --- nonwords constructed to match strong regularities, for example *nust*, which is completely regular (e.g., *must*, *bust*, *trust*, etc). 120 | 121 | * **Glushko exceptions** --- nonwords that have similar English exceptions and conflicting regularities, such as *bint* (could be like *mint*, but also could be like *pint*). We score these items either according to the predominant regularity, or also including close exceptional cases (alt OK in the table). 122 | 123 | * **McCann & Besner ctrls** --- these are pseudo-homophones and matched controls, that sound like actual words, but are spelled in a novel way, for example *choyce* (pronounced like *choice*), and the matched control is *phoyce*. 124 | 125 | * **Taraban & McClelland** --- has frequency matched regular and exception nonwords, for example *poes* (like high frequency words *goes* or *does*), and *mose*, like lower frequency *pose* or *lose*. 126 | 127 | The results indicate that the model does a remarkably good job of capturing the performance of people's performance on these nonword reading sets. This suggests that the model is capable of learning the appropriate regularities and subregularities that are present in the statistics of English pronunciation. 128 | 129 | | Nonword Set | ss Model | PMSP | People | 130 | |---------------------------|----------|-------|--------| 131 | | Glushko regulars | 95.3 | 97.7 | 93.8 | 132 | | Glushko exceptions raw | 79.0 | 72.1 | 78.3 | 133 | | Glushko exceptions alt OK | 97.6 | 100.0 | 95.9 | 134 | | McCann & Besner ctrls | 85.9 | 85.0 | 88.6 | 135 | | McCann & Besner homoph | 92.3 | n/a | 94.3 | 136 | | Taraban & McClelland | 97.9 | n/a | 100.0 | 137 | 138 | Table: Comparison of nonword reading performance for our spelling-to-sound model (ss Model), the PMSP model, and data from people, across a range of different nonword datasets as described in the text. Our model performs comparably to people, after learning on nearly 3,000 English monosyllabic words. {#tbl:table-nonword} 139 | 140 | ### Exploration 141 | 142 | Run `ss` (spelling to sound) in [CCN Sims](https://compcogneuro.org/simulations) to explore the spelling-to-sound model, and test its performance on both word and nonword stimuli. 143 | 144 | ## Reading and Dyslexia in the Triangle Model 145 | 146 | ![Triangle model of reading pathways: Visual word input (orthography) can produce speech output of the word either directly via projections to phonology (direct path), or indirectly via projections to semantics that encode the meanings of words. There is no single "lexicon" of words in this model --- word representations are instead distributed across these different pathways. Damage to different pathways can account for properties of acquired dyslexia.](figures/fig_lang_paths.png){#fig:fig-lang-paths width=40% } 147 | 148 | The next language model we explore brings together the components of the semantics and spelling-to-sound models to simulate the major pathways involved in reading, according to the so-called *triangle model* ([@fig:fig-lang-paths]) [@PlautShallice93]. This model provides a basic understanding of the functional roles of visual perception of written words (*orthography*), spoken motor output of word *phonology*, and *semantic* representations of word meaning in between. This set of language pathways is sufficient to simulate the processes involved in reading words aloud, and damage to these pathways can simulate the critical features of different types of acquired dyslexia. Acquired dyslexia, which results from strokes or other brain damage, is distinct from *developmental* dyslexia, which is the more common form that many people associate with the term *dyslexia* (which generically refers to any form of reading impairment). 149 | 150 | There are three major forms of acquired dyslexia that can be simulated with the model: 151 | 152 | * **Phonological** --- characterized by difficulty reading nonwords (e.g., "nust" or "mave"). This can be produced by damage to the direct pathway between orthography and phonology (there shouldn't be any activation in semantics for nonwords), such that people have difficulty mapping spelling to sound according to learned regularities that can be applied to nonwords. We'll explore this phenomenon in greater detail in the next simulation. 153 | 154 | * **Deep** --- is a more severe form of phonological dyslexia, with the striking feature that people sometimes make semantic substitutions for words, pronouncing the word "orchestra" as "symphony" for example. There are also *visual* errors, so-named because they seem to reflect a misperception of the word inputs (e.g, reading the word "dog" as "dot"). Interestingly, we'll see how more significant damage to the direct pathway can give rise to this profile --- the semantic errors occur due to everything going through the semantic layer, such that related semantic representations can be activated. In the normal intact brain, the direct pathway provides the relevant constraints to produce the actual written word, but absent this constraint, an entirely different but semantically related word can be output. 155 | 156 | * **Surface** --- here nonword reading is intact, but access to semantics is impaired (as in Wernicke's aphasia), strongly implicating a lesion in the semantics pathway. Interestingly, pronunciation of exception words (e.g., "yacht") is impaired. This suggests that people typically rely on the semantic pathway to "memorize" how to pronounce odd words like yacht, and the direct pathway is used more for pronouncing regular words. 157 | 158 | That these different forms of dyslexia can be reliably observed in different patients, and fit so well with expected patterns of reading deficits according to the triangle model, provides a strong source of support for the validity of the model. It would be even more compelling if the specific foci of damage associated with these different forms of dyslexia make sense anatomically according to the mapping of the triangle model onto brain areas. 159 | 160 | ### Exploration 161 | 162 | ![Cluster plot of semantic similarity for words in the simple triangle model of reading and dyslexia. Words that are semantically close (e.g., within the same terminal cluster) are sometimes confused for each other in simulated deep dyslexia.](figures/fig_dyslex_sem_clust.png){#fig:fig-dyslex-sem-clust width=50% } 163 | 164 | Run `dyslexia` from [CCN Sims](https://compcogneuro.org/simulations) for the simulation of the triangle model and associated forms of dyslexia. This model allows you to simulate the different forms of acquired dyslexia, in addition to normal reading, using the small corpus of words as shown in [@fig:fig-dyslex-sem-clust]. 165 | 166 | ## Syntax and Semantics in a Sentence Gestalt 167 | 168 | ![Syntactic diagram of a basic sentence. S = sentence; NP = noun phrase; Art = article; N = noun; VP = verb phrase; V = verb.](figures/fig_lang_phrase_stru.png){#fig:fig-lang-phrase-stru width=30% } 169 | 170 | Having covered some of the interesting properties of language at the level of individual words, we now take one step higher, to the level of sentences. This step brings us face-to-face with the thorny issue of *syntax*. The traditional approach to syntax assumes that people assemble something akin to those tree-like syntactic structures you learned (or maybe not) in school ([@fig:fig-lang-phrase-stru]). But given that these things need to be explicitly taught, and don't seem to be the most natural way of thinking for many people, it seems perhaps unlikely that this is how our brains actually process language. 171 | 172 | These syntactic structures also assume a capacity for role-filler binding that is actually rather challenging to achieve in neural networks. For example, the assumption is that you somehow "bind" the noun *boy* into a variable slot that is designated to contain the subject of the sentence. And once you move on to the next sentence, this binding is replaced with the next one. This constant binding and unbinding is rather like the rotation of a wheel on a car --- it tends to rip apart anything that might otherwise try to attach to the wheel. One important reason people have legs instead of wheels is that we need to provide those legs with a blood supply, nerves, etc, all of which could not survive the rotation of a wheel. Similarly, our neurons thrive on developing longer-term stable connections via physical synapses, and are not good at this rapid binding and unbinding process. We focus on these issues in greater depth in the *Executive Function* Chapter. 173 | 174 | An alternative way of thinking about sentence processing that is based more directly on neural network principles is captured in the **Sentence Gestalt** model of [@StJohnMcClelland90]. The key idea is that both syntax and semantics merge into an evolving distributed representation that captures the overall gestalt meaning of a sentence, without requiring all the precise syntactic bindings assumed in the traditional approach. We don't explicitly bind *boy* to subject, but rather encode the larger meaning of the overall sentence, which implies that the boy is the subject (or more precisely, the *agent*), because he is doing the chasing. 175 | 176 | One advantage of this way of thinking is that it more naturally deals with all the ambiguity surrounding the process of parsing syntax, where the specific semantics of the collection of words can dramatically alter the syntactic interpretation. A classic demonstration of this ambiguity is the sentence: 177 | 178 | > *Time flies like an arrow.* 179 | 180 | which may not seem very ambiguous, until you consider alternatives, such as: 181 | 182 | > *Fruit flies like a banana.* 183 | 184 | The word *flies* can be either a verb or noun depending on the semantic context. Further reflection reveals several more ambiguous interpretations of the first sentence, which are fun to have take hold over your brain as you re-read the sentence. Another example from [@Rohde02] is: 185 | 186 | > *The slippers were found by the nosy dog.* 187 | 188 | > *The slippers were found by the sleeping dog.* 189 | 190 | just a single subtle word change recasts the entire meaning of the sentence, from one where the dog is the agent to one where it plays a more peripheral role. 191 | 192 | If you don't bother with the syntactic parse in the first place, and just try to capture the meaning of the sentence, then none of this ambiguity really matters. The meaning of a sentence is generally much less ambiguous than the syntactic parse --- getting the syntax exactly right requires making a lot of fine-grained distinctions that people may not actually bother with. But the meaning does depend on the exact combination of words, so there is a lot of emergent meaning in a sentence --- here's another example from [@Rohde02] where the two sentences are syntactically identical but have very different meaning: 193 | 194 | > *We finally put the baby to sleep.* 195 | 196 | > *We finally put the dog to sleep.* 197 | 198 | The notion of a semantically-oriented gestalt representation of a sentence seems appealing, but until an implemented model actually shows that such a thing actually works, it is all just a nice story. The St. John & McClelland (1990) model does demonstrate that a distributed representation formed incrementally as words are processed in a sentence can then be used to answer various comprehension questions about that sentence. However, it does so using a very small space of language, and it is not clear how well it generalizes to new words, or scales to a more realistically complex language. A more sophisticated model by [@Rohde02] that adopts a similar overall strategy does provide some promise for positive answers to these challenges. The training of the the Rohde model uses structured semantic representations in the form of slot-filler propositions about the thematic roles of various elements of the sentence. These include the roles: agent, experiencer, goal, instrument, patient, source, theme, beneficiary, companion, location, author, possession, subtype, property, if, because, while, and although. This thematic role binding approach is widely used in the natural language processing field for encoding semantics, but it moves away from the notion of an unstructured gestalt representation of semantic meaning. The sentence gestalt model uses a much simpler form of this thematic role training, which seems less controversial in this respect. 199 | 200 | ### The Sentence Gestalt Model 201 | 202 | The sentence gestalt (SG) model is trained on a very small toy world, consisting of the following elements: 203 | 204 | * People: *busdriver* (adult male), *teacher*, (adult female), *schoolgirl*, *pitcher* (boy). *adult, child, someone* also used. 205 | 206 | * Actions: *eat, drink, stir, spread, kiss, give, hit, throw, drive, rise*. 207 | 208 | * Objects: *spot* (the dog), *steak, soup, ice cream, crackers, jelly, iced tea, kool aid, spoon, knife, finger, rose, bat* (animal), *bat* (baseball), *ball* (sphere), *ball* (party), *bus, pitcher, fur*. 209 | 210 | * Locations: *kitchen, living room, shed, park*. 211 | 212 | The semantic roles used to probe the network during training are: *agent, action, patient, instrument, co-agent, co-patient, location, adverb, recipient.* 213 | 214 | The main syntactic variable is the presence of active vs. passive construction, and clauses that further specify events. Also, as you can see, several of the words are ambiguous so that context must be used to disambiguate. 215 | 216 | The model is trained on randomly-generated sentences according to a semantic and syntactic grammar that specifies which words tend to co-occur etc. It is then tested on a set of key test sentences to probe its behavior in various ways: 217 | 218 | * Active semantic: *The schoolgirl stirred the kool-aid with a spoon.* (kool-aid can only be the patient, not the agent of this sentence) 219 | 220 | * Active syntactic: *The busdriver gave the rose to the teacher.* (teacher could be either patient or agent --- word order syntax determines it). 221 | 222 | * Passive semantic: *The jelly was spread by the busdriver with the knife.* (jelly can't be agent, so must be patient) 223 | 224 | * Passive syntactic: *The teacher was kissed by the busdriver.* vs. *The busdriver kissed the teacher.* (either teacher or busdriver could be agent, syntax alone determines which it is). 225 | 226 | * Word ambiguity: *The busdriver threw the ball in the park.*, *The teacher threw the ball in the living room.* (ball is ambiguous, but semantically, busdriver throws balls in park, while teacher throws balls in living room) 227 | 228 | * Concept instantiation: *The teacher kissed someone.* (male). (teacher always kisses a male --- has model picked up on this?) 229 | 230 | * Role elaboration: *The schoolgirl ate crackers.* (with finger); *The schoolgirl ate.* (soup) (these are predominant cases) 231 | 232 | * Online update: *The child ate soup with daintiness.* vs. *The pitcher ate soup with daintiness.* (schoolgirl usually eats soup, so ambiguous *child* is resolved as schoolgirl in first case after seeing soup, but specific input of *pitcher* in second case prevents this updating). 233 | 234 | * Conflict: *The adult drank iced-tea in the kitchen.* (living-room) (iced-tea is always had in the living room). 235 | 236 | ![The sentence gestalt model, implemented in the DeepLeabra framework that does predictive learning with deep layer (D suffix) temporal context representations, that function like the context layer in a standard SRN (simple recurrent network). A single word at a time is presented in the input, which is encoded into the gestalt layer. The gestalt deep context layer (`GestaltD`) effectively maintains a copy of the gestalt from the previous time step, enabling integration of information across words in the sentence. This gestalt is probed by querying the semantic role, with training based on the ability to produce the correct filler output.](figures/fig_sg_net_deepleabra.png){#fig:fig-sg-net-deepleabra width=50% } 237 | 238 | The model structure ([@fig:fig-sg-net-deepleabra]) has single word inputs (using localist single-unit representations of words) projecting up through an encoding hidden layer to the gestalt layer, which is where the distributed representation of sentence meaning develops. The memory for prior words and meaning interpretations of the sentence is encoded via a context layer, which effectively retains a copy of the gestalt layer activation state from the previous word input. Historically, this context layer is known as a **simple recurrent network (SRN)**, and it is widely used in neural network models of temporally extended tasks [@Elman90; @Jordan89; @CleeremansMcClelland91; @MunakataMcClellandJohnsonEtAl97]. In this model, we are using a biologically-based version of the SRN, based on the thalamocortical connections between the Pulvinar nucleus of the thalamus and the deep layers of the cortex [@OReillyWyatteRohrlich17], which is implemented in the *DeepLeabra* version of Leabra. The network training comes from repeated probing of the network for the various semantic roles enumerated above (e.g., *agent* vs. *patient*). A role input unit is activated, and then the network is trained to activate the appropriate response in the filler output layer. In addition, as in other DeepLeabra models (and other SRN models), the encoder layer attempts to predict the next input, and learns from errors in these predictions. 239 | 240 | ![Cluster plot over the gestalt layer of patterns associated with the different nouns, showing that these distributed representations capture the semantic similarities of the words (much as in the LSA-like semantics model explored in the previous section).](figures/fig_sg_noun_clust_deep.png){#fig:fig-sg-noun-clust-deep width=60% } 241 | 242 | ![Cluster plot over the gestalt layer of patterns associated with a set of test sentences designed to test for appropriate similarity relationships. sc = schoolgirl; st = stirred; ko = kool-aid; te = teacher; bu= busddriver; pi = pitcher; dr = drank; ic = iced-tea; at = ate; so = soup; st = steak.](figures/fig_sg_sent_probe_clust_deep.png){#fig:fig-sg-sent-probe-clust-deep width=50% } 243 | 244 | [@fig:fig-sg-noun-clust-deep] shows a cluster plot of the gestalt layer representations of the different nouns, indicating that the network does develop sensible semantic similarity structure for these words. Probing further, [@fig:fig-sg-sent-probe-clust-deep] shows the cluster plot for a range of related sentences, indicating a sensible verb-centric semantic organization --- sentences sharing the same verb are all clustered together, and then agents within that form a second level of organization. 245 | 246 | ### Exploration 247 | 248 | Run the `sg` model from [CCN Sims](https://compcogneuro.org/simulations) to explore the sentence gestalt model. 249 | 250 | ## Next Steps in Language Modeling of Sentences and Beyond 251 | 252 | The primary function of language is to communicate. It is fundamentally about semantics. And semantics represents a major barrier to further progress in language modeling. The sentence gestalt model has very simplistic semantics, and the more advanced version of it developed by [@Rohde02] introduces more complex semantics, at the cost of injecting externally more of what the model should be developing on its own. Thus, the fundamental challenge for models of sentence-level or higher-level language is to develop a more naturalistic way of training the corresponding semantics. In an ideal case, a virtual humanoid robot would be wandering around a rich simulated naturalistic environment, and receiving and producing language in order to understand and survive in this environment. This would mimic the way in which people acquire and use language, and would undoubtedly provide considerable insight into the nature of language acquisition and higher-level semantic representations. But clearly this will require a lot of work. 253 | 254 | 255 | -------------------------------------------------------------------------------- /cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/cover.png -------------------------------------------------------------------------------- /cover.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 27 | 28 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | Co 83 | Co 84 | m 85 | m 86 | pu 87 | pu 88 | t 89 | t 90 | ational 91 | ational 92 | 93 | 94 | Cogniti 95 | Cogniti 96 | v 97 | v 98 | e 99 | e 100 | 101 | 102 | N 103 | N 104 | eu 105 | eu 106 | r 107 | r 108 | oscience 109 | oscience 110 | 5 111 | 5 112 | t 113 | t 114 | h Edition 115 | h Edition 116 | R 117 | andall C. O' 118 | R 119 | eil 120 | l 121 | y 122 | Y 123 | u 124 | k 125 | o Muna 126 | k 127 | a 128 | t 129 | a 130 | Mi 131 | c 132 | hael J. 133 | F 134 | r 135 | ank 136 | Thomas E. Hazy 137 | and Contributors 138 | 139 | -------------------------------------------------------------------------------- /endmatter.md: -------------------------------------------------------------------------------- 1 | # Acknowledgments {-} 2 | 3 | We thank the large number of students who have helped improve this textbook over the years, in addition to the support and understanding of our families. We thank Kai O'Reilly for his development of [Cogent Core](https://www.cogentcore.org/core/), which supports the simulations and website that go with this book. 4 | 5 | \ 6 | 7 | # About the Authors {-} 8 | 9 |       Randall C. O'Reilly is a Scientist at Obelisk Lab in the Astera Institute, and Adjunct Professor of Psychology and Computer Science at the Center for Neuroscience at the University of California, Davis. [https://ccnlab.org/people/oreilly](https://ccnlab.org/people/oreilly) 10 | 11 | Yuko Munakata is Professor of Psychology in the Center for Mind and Brain at the University of California, Davis and Research Affiliate at the University of Colorado Boulder. 12 | \ 13 | [https://psychology.ucdavis.edu/people/yuko-munakata](https://psychology.ucdavis.edu/people/yuko-munakata) 14 | 15 | Michael J Frank is Edgar L. Marston Professor of Cognitive and Psychological Sciences and the Director of the Carney Center for Computational Brain Science at Brown University. 16 | \ 17 | [https://ski.clps.brown.edu/](https://ski.clps.brown.edu/) 18 | 19 | Thomas E. Hazy is a Research Scientist at Obelisk Lab in the Astera Institute. 20 | 21 | Many others have contributed to this text. 22 | 23 | -------------------------------------------------------------------------------- /figures/fig_Fuster01_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_Fuster01_2.png -------------------------------------------------------------------------------- /figures/fig_SommerWurtz00_fig2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_SommerWurtz00_fig2.png -------------------------------------------------------------------------------- /figures/fig_ab_ac_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_ab_ac_list.png -------------------------------------------------------------------------------- /figures/fig_actor_critic_basic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_actor_critic_basic.png -------------------------------------------------------------------------------- /figures/fig_adex_vm_vs_hz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_adex_vm_vs_hz.png -------------------------------------------------------------------------------- /figures/fig_aphasia_areas.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_aphasia_areas.png -------------------------------------------------------------------------------- /figures/fig_attractor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_attractor.png -------------------------------------------------------------------------------- /figures/fig_attractor_cal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_attractor_cal.png -------------------------------------------------------------------------------- /figures/fig_bcm_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bcm_function.png -------------------------------------------------------------------------------- /figures/fig_bg_action_sel_dam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bg_action_sel_dam.png -------------------------------------------------------------------------------- /figures/fig_bg_frontal_da_burst_dip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bg_frontal_da_burst_dip.png -------------------------------------------------------------------------------- /figures/fig_bg_gating_science.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bg_gating_science.png -------------------------------------------------------------------------------- /figures/fig_bg_loops_ads86.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bg_loops_ads86.png -------------------------------------------------------------------------------- /figures/fig_bg_opal_hilow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bg_opal_hilow.png -------------------------------------------------------------------------------- /figures/fig_bidir_backprop_intuition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bidir_backprop_intuition.png -------------------------------------------------------------------------------- /figures/fig_bp_compute_delta.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bp_compute_delta.png -------------------------------------------------------------------------------- /figures/fig_brain_anatomy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_brain_anatomy.png -------------------------------------------------------------------------------- /figures/fig_brain_and_lobes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_brain_and_lobes.png -------------------------------------------------------------------------------- /figures/fig_brain_puzzle_behav_bio_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_brain_puzzle_behav_bio_data.png -------------------------------------------------------------------------------- /figures/fig_brain_puzzle_blue_sky_pieces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_brain_puzzle_blue_sky_pieces.png -------------------------------------------------------------------------------- /figures/fig_brodmann_areas.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_brodmann_areas.png -------------------------------------------------------------------------------- /figures/fig_brodmann_areas_3d_color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_brodmann_areas_3d_color.png -------------------------------------------------------------------------------- /figures/fig_brodmann_areas_color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_brodmann_areas_color.png -------------------------------------------------------------------------------- /figures/fig_bvpvlv_functional_org_simpler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bvpvlv_functional_org_simpler.png -------------------------------------------------------------------------------- /figures/fig_bvpvlv_sim1a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_bvpvlv_sim1a.png -------------------------------------------------------------------------------- /figures/fig_category_hierarch_dist_reps.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_category_hierarch_dist_reps.png -------------------------------------------------------------------------------- /figures/fig_ccnbook_cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_ccnbook_cover.png -------------------------------------------------------------------------------- /figures/fig_cerebellum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cerebellum.png -------------------------------------------------------------------------------- /figures/fig_cerebellum_cortex_inputs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cerebellum_cortex_inputs.png -------------------------------------------------------------------------------- /figures/fig_coarse_coding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_coarse_coding.png -------------------------------------------------------------------------------- /figures/fig_cortex_bidir_cons_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortex_bidir_cons_map.png -------------------------------------------------------------------------------- /figures/fig_cortex_bio_arealayers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortex_bio_arealayers.png -------------------------------------------------------------------------------- /figures/fig_cortex_bio_layers.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortex_bio_layers.jpg -------------------------------------------------------------------------------- /figures/fig_cortex_bio_neurtypes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortex_bio_neurtypes.png -------------------------------------------------------------------------------- /figures/fig_cortex_lobes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortex_lobes.png -------------------------------------------------------------------------------- /figures/fig_cortical_cons_ff_fb_lat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortical_cons_ff_fb_lat.png -------------------------------------------------------------------------------- /figures/fig_cortical_fun_org_tins_acc_ofc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortical_fun_org_tins_acc_ofc.png -------------------------------------------------------------------------------- /figures/fig_cortical_layers_in_hid_out.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortical_layers_in_hid_out.png -------------------------------------------------------------------------------- /figures/fig_cortical_lobes.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortical_lobes.jpg -------------------------------------------------------------------------------- /figures/fig_cortical_neuron.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_cortical_neuron.png -------------------------------------------------------------------------------- /figures/fig_dalmatian.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_dalmatian.png -------------------------------------------------------------------------------- /figures/fig_diffusion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_diffusion.png -------------------------------------------------------------------------------- /figures/fig_dist_rep_vis_bio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_dist_rep_vis_bio.png -------------------------------------------------------------------------------- /figures/fig_distrib_sem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_distrib_sem.png -------------------------------------------------------------------------------- /figures/fig_dyslex_sem_clust.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_dyslex_sem_clust.png -------------------------------------------------------------------------------- /figures/fig_electricity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_electricity.png -------------------------------------------------------------------------------- /figures/fig_expect_outcome_errs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_expect_outcome_errs.png -------------------------------------------------------------------------------- /figures/fig_face_categ_dim_prjn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_face_categ_dim_prjn.png -------------------------------------------------------------------------------- /figures/fig_felleman_vanessen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_felleman_vanessen.png -------------------------------------------------------------------------------- /figures/fig_frank_badre12_pfc_hierarch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_frank_badre12_pfc_hierarch.png -------------------------------------------------------------------------------- /figures/fig_gears.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_gears.png -------------------------------------------------------------------------------- /figures/fig_generec_compute_delta.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_generec_compute_delta.png -------------------------------------------------------------------------------- /figures/fig_halle_berry_neuron.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_halle_berry_neuron.jpg -------------------------------------------------------------------------------- /figures/fig_hasselmo_bodelon_wyble_theta.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_hasselmo_bodelon_wyble_theta.png -------------------------------------------------------------------------------- /figures/fig_haxbyetal01_obj_maps.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_haxbyetal01_obj_maps.jpg -------------------------------------------------------------------------------- /figures/fig_hcmp_rfs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_hcmp_rfs.png -------------------------------------------------------------------------------- /figures/fig_hebb_demo_blank.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_hebb_demo_blank.png -------------------------------------------------------------------------------- /figures/fig_hikosaka_gating.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_hikosaka_gating.png -------------------------------------------------------------------------------- /figures/fig_hippo_comp_learn_sys.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_hippo_comp_learn_sys.png -------------------------------------------------------------------------------- /figures/fig_hippo_mem_formation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_hippo_mem_formation.png -------------------------------------------------------------------------------- /figures/fig_inhib_types.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_inhib_types.png -------------------------------------------------------------------------------- /figures/fig_integ_lrate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_integ_lrate.png -------------------------------------------------------------------------------- /figures/fig_invar_trans.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_invar_trans.png -------------------------------------------------------------------------------- /figures/fig_ions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_ions.png -------------------------------------------------------------------------------- /figures/fig_ipa_chart_consonants_simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_ipa_chart_consonants_simple.png -------------------------------------------------------------------------------- /figures/fig_ipa_chart_vowels_ipa_pmsp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_ipa_chart_vowels_ipa_pmsp.png -------------------------------------------------------------------------------- /figures/fig_kirkwood_et_al_96_bcm_thresh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_kirkwood_et_al_96_bcm_thresh.png -------------------------------------------------------------------------------- /figures/fig_kobatake_tanaka_94_invariance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_kobatake_tanaka_94_invariance.png -------------------------------------------------------------------------------- /figures/fig_kobatake_tanaka_94_v2_v4_it.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_kobatake_tanaka_94_v2_v4_it.png -------------------------------------------------------------------------------- /figures/fig_lang_paths.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_lang_paths.png -------------------------------------------------------------------------------- /figures/fig_lang_phrase_stru.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_lang_phrase_stru.png -------------------------------------------------------------------------------- /figures/fig_leabra_mechs_xcal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_leabra_mechs_xcal.png -------------------------------------------------------------------------------- /figures/fig_leutgeb_et_al_07_dg_patsep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_leutgeb_et_al_07_dg_patsep.png -------------------------------------------------------------------------------- /figures/fig_location_terms.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_location_terms.png -------------------------------------------------------------------------------- /figures/fig_lookup_table_function_approximator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_lookup_table_function_approximator.png -------------------------------------------------------------------------------- /figures/fig_ltp_ltd_ca2+.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_ltp_ltd_ca2+.png -------------------------------------------------------------------------------- /figures/fig_ltpd_synapse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_ltpd_synapse.png -------------------------------------------------------------------------------- /figures/fig_markov_et_al_cort_hier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_markov_et_al_cort_hier.png -------------------------------------------------------------------------------- /figures/fig_mnt2out.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_mnt2out.png -------------------------------------------------------------------------------- /figures/fig_neglect_drawings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_neglect_drawings.png -------------------------------------------------------------------------------- /figures/fig_neglect_line_bisect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_neglect_line_bisect.png -------------------------------------------------------------------------------- /figures/fig_neglect_portrait.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_neglect_portrait.jpg -------------------------------------------------------------------------------- /figures/fig_neuron_as_detect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_neuron_as_detect.png -------------------------------------------------------------------------------- /figures/fig_neuron_rate_code_approx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_neuron_rate_code_approx.png -------------------------------------------------------------------------------- /figures/fig_neuron_spiking.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_neuron_spiking.png -------------------------------------------------------------------------------- /figures/fig_nxx1_fun.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_nxx1_fun.png -------------------------------------------------------------------------------- /figures/fig_objrec_difficulty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_objrec_difficulty.png -------------------------------------------------------------------------------- /figures/fig_objrec_objs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_objrec_objs.png -------------------------------------------------------------------------------- /figures/fig_on_off_rfields_edge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_on_off_rfields_edge.png -------------------------------------------------------------------------------- /figures/fig_patsep_clr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_patsep_clr.png -------------------------------------------------------------------------------- /figures/fig_pbwm_architecture_bio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_pbwm_architecture_bio.png -------------------------------------------------------------------------------- /figures/fig_pbwm_architecture_mnt_out.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_pbwm_architecture_mnt_out.png -------------------------------------------------------------------------------- /figures/fig_pfc_trc_reverb_loops.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_pfc_trc_reverb_loops.png -------------------------------------------------------------------------------- /figures/fig_posner_disengage_mech.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_posner_disengage_mech.png -------------------------------------------------------------------------------- /figures/fig_posner_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_posner_graph.png -------------------------------------------------------------------------------- /figures/fig_posner_task.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_posner_task.png -------------------------------------------------------------------------------- /figures/fig_pvlv_bio_no_cereb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_pvlv_bio_no_cereb.png -------------------------------------------------------------------------------- /figures/fig_pvlv_da.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_pvlv_da.png -------------------------------------------------------------------------------- /figures/fig_reading_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_reading_model.png -------------------------------------------------------------------------------- /figures/fig_schultz97_vta_td.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_schultz97_vta_td.png -------------------------------------------------------------------------------- /figures/fig_sg_net_deepleabra.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_sg_net_deepleabra.png -------------------------------------------------------------------------------- /figures/fig_sg_noun_clust_deep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_sg_noun_clust_deep.png -------------------------------------------------------------------------------- /figures/fig_sg_sent_probe_clust_deep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_sg_sent_probe_clust_deep.png -------------------------------------------------------------------------------- /figures/fig_spat_attn_lateral.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_spat_attn_lateral.png -------------------------------------------------------------------------------- /figures/fig_stdp_bipoo98.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_stdp_bipoo98.png -------------------------------------------------------------------------------- /figures/fig_stroop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_stroop.png -------------------------------------------------------------------------------- /figures/fig_stroop_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_stroop_data.png -------------------------------------------------------------------------------- /figures/fig_synapse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_synapse.png -------------------------------------------------------------------------------- /figures/fig_synapse_em.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_synapse_em.jpg -------------------------------------------------------------------------------- /figures/fig_tanaka03_topo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_tanaka03_topo.png -------------------------------------------------------------------------------- /figures/fig_tanaka03_topo_maps.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_tanaka03_topo_maps.png -------------------------------------------------------------------------------- /figures/fig_urakubo_et_al_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_urakubo_et_al_model.png -------------------------------------------------------------------------------- /figures/fig_v1_orientation_cols_data.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_v1_orientation_cols_data.jpg -------------------------------------------------------------------------------- /figures/fig_v1_orientation_tuning_data.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_v1_orientation_tuning_data.jpg -------------------------------------------------------------------------------- /figures/fig_v1_simple_dog_edge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_v1_simple_dog_edge.png -------------------------------------------------------------------------------- /figures/fig_v1_visual_filters_hypercols_basic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_v1_visual_filters_hypercols_basic.png -------------------------------------------------------------------------------- /figures/fig_v1rf_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_v1rf_map.png -------------------------------------------------------------------------------- /figures/fig_vert_line_detector_probs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_vert_line_detector_probs.png -------------------------------------------------------------------------------- /figures/fig_vis_optic_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_vis_optic_path.png -------------------------------------------------------------------------------- /figures/fig_vis_system_bio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_vis_system_bio.png -------------------------------------------------------------------------------- /figures/fig_vm_as_tug_of_war.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_vm_as_tug_of_war.png -------------------------------------------------------------------------------- /figures/fig_vm_as_tug_of_war_cases.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_vm_as_tug_of_war_cases.png -------------------------------------------------------------------------------- /figures/fig_vocal_tract.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_vocal_tract.png -------------------------------------------------------------------------------- /figures/fig_wheres_waldo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_wheres_waldo.jpg -------------------------------------------------------------------------------- /figures/fig_wt_contrast_sigmoid_fun.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_wt_contrast_sigmoid_fun.png -------------------------------------------------------------------------------- /figures/fig_xcal_bcm_err_learn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_xcal_bcm_err_learn.png -------------------------------------------------------------------------------- /figures/fig_xcal_bcm_selforg_learn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_xcal_bcm_selforg_learn.png -------------------------------------------------------------------------------- /figures/fig_xcal_dwt_fun.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_xcal_dwt_fun.png -------------------------------------------------------------------------------- /figures/fig_xcal_dwt_fun_urakubo_fit_full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/fig_xcal_dwt_fun_urakubo_fit_full.png -------------------------------------------------------------------------------- /figures/web_header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CompCogNeuro/book/e3195859d82c581ceebc0d83176f26df0ba27577/figures/web_header.png -------------------------------------------------------------------------------- /frontmatter.md: -------------------------------------------------------------------------------- 1 | #### Copyright {-} 2 | 3 |      This is an open-source textbook, available at [https://compcogneuro.org/book](https://compcogneuro.org/book) and hosted at [https://github.com/CompCogNeuro/book](https://github.com/CompCogNeuro/book). This is the fifth edition of this book. 4 | 5 | The hands-on simulations to explore the models described in this book are available at [https://compcogneuro.org/simulations](https://compcogneuro.org/simulations) and hosted at [https://github.com/CompCogNeuro/sims](https://github.com/CompCogNeuro/sims) 6 | 7 | Copyright © 2024 Randall C. O'Reilly, Yuko Munakata, Michael J. Frank, Thomas E. Hazy, and Contributors 8 | 9 | All rights reserved. Book licensed under [CC-BY-4.0](https://github.com/CompCogNeuro/book/blob/main/LICENSE). Simulations licensed under [BSD 3-Clause](https://github.com/CompCogNeuro/sims/blob/main/LICENSE). 10 | 11 | # Dedication {-} 12 | 13 | _To our families._ 14 | 15 | 16 | -------------------------------------------------------------------------------- /glossary.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /metadata.yaml: -------------------------------------------------------------------------------- 1 | copyright: 2 | owner: "Randall C. O'Reilly, Yuko Munakata, Michael J. Frank, Thomas E. Hazy, and Contributors" 3 | year: "2024" 4 | version: "v1.1.1" 5 | 6 | documentclass: book 7 | top-level-division: chapter 8 | chapters: true 9 | number-sections: true 10 | 11 | title: "Computational Cognitive Neuroscience" 12 | 13 | publisher: "Open Textbook, freely available" 14 | 15 | # Multiple authors are permitted. 16 | author: 17 | - "Randall C. O'Reilly" 18 | - Yuko Munakata 19 | - Michael J. Frank 20 | - Thomas E. Hazy 21 | - Contributors 22 | 23 | # Book identifier: 24 | identifier_scheme: ISBN 25 | identifier: "9798336823622" 26 | 27 | # The language and variant in which the book is written. 28 | language: en-US 29 | 30 | # See https://wiki.mobileread.com/wiki/Genre 31 | genre: reference 32 | 33 | # citation styling -- get from https://github.com/citation-style-language/styles 34 | csl: apa-6th-edition.csl 35 | 36 | # Set this option if you need to change the paper size. Default is "letter". 37 | # "A4" is also valid. 38 | papersize: letter 39 | 40 | urlcolor: "blue" 41 | 42 | link-citations: true 43 | 44 | # align display equations on the left 45 | classoption: fleqn 46 | 47 | subparagraph: true 48 | 49 | linkReferences: true 50 | # note: set in html-metadata 51 | # numberSections: false 52 | sectionDepth: 0 53 | 54 | figPrefix: 55 | - "Figure" 56 | - "Figures" 57 | 58 | tblPrefix: 59 | - "Table" 60 | - "Tables" 61 | 62 | secPrefix: 63 | - "Chapter" 64 | - "Chapters" 65 | 66 | 67 | --------------------------------------------------------------------------------